From b936ed0055e364ed652dc7def086124e90035455 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:41:29 -0400 Subject: [PATCH 01/14] OUT-2: Opus (CELT) encoder (effectcraft-opusenc) New L0 crate effectcraft-opusenc: clean-room RFC 6716 CELT-only encoder (fullband 48 kHz, 20 ms packets, mono/stereo as dual L/R stereo, constant packet size from the bitrate) with the RFC 7845 OpusHead and pre-skip. Forward low-overlap MDCT, inter/intra coarse energy, fine/final energy, the decoder's exact allocation, recursive band splitting, PVQ search and codeword indexing, silence frames. crates/export depends on it; filmcraft-opus (dev) is the round-trip oracle in tests/opus_roundtrip.rs, with ffmpeg reading an in-test Ogg Opus file as a second oracle when installed. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 6 + crates/export/Cargo.toml | 2 + crates/export/tests/opus_roundtrip.rs | 364 +++++++++++++++++ crates/opusenc/Cargo.toml | 12 + crates/opusenc/README.md | 33 ++ crates/opusenc/src/bands.rs | 497 +++++++++++++++++++++++ crates/opusenc/src/energy.rs | 230 +++++++++++ crates/opusenc/src/lib.rs | 311 ++++++++++++++ crates/opusenc/src/mdct.rs | 293 ++++++++++++++ crates/opusenc/src/range.rs | 472 ++++++++++++++++++++++ crates/opusenc/src/rate.rs | 561 ++++++++++++++++++++++++++ crates/opusenc/src/tables.rs | 122 ++++++ docs/architecture.md | 1 + xtask/src/main.rs | 1 + 15 files changed, 2911 insertions(+) create mode 100644 crates/export/tests/opus_roundtrip.rs create mode 100644 crates/opusenc/Cargo.toml create mode 100644 crates/opusenc/README.md create mode 100644 crates/opusenc/src/bands.rs create mode 100644 crates/opusenc/src/energy.rs create mode 100644 crates/opusenc/src/lib.rs create mode 100644 crates/opusenc/src/mdct.rs create mode 100644 crates/opusenc/src/range.rs create mode 100644 crates/opusenc/src/rate.rs create mode 100644 crates/opusenc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..1a1f0f604 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1218,6 +1218,7 @@ version = "0.1.0" dependencies = [ "effectcraft-color", "effectcraft-media", + "effectcraft-opusenc", "effectcraft-project", "effectcraft-raster", "effectcraft-render", @@ -1225,6 +1226,7 @@ dependencies = [ "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", + "filmcraft-opus", "filmcraft-prores", "gif", "image", @@ -1338,6 +1340,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-opusenc" +version = "0.1.0" + [[package]] name = "effectcraft-path" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf45..da4f7047c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-opusenc = { path = "crates/opusenc" } # foundation thiserror = "2" @@ -66,6 +67,8 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -129,6 +132,9 @@ opt-level = 3 [profile.dev.package.effectcraft-media] opt-level = 3 +[profile.dev.package.effectcraft-opusenc] +opt-level = 3 + [profile.dev.package.effectcraft-export] opt-level = 3 diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 34c83aabd..5bdd0fee6 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -18,6 +18,7 @@ filmcraft-h264enc = { workspace = true } filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +effectcraft-opusenc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -28,3 +29,4 @@ log = { workspace = true } [dev-dependencies] effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } +filmcraft-opus = { workspace = true } diff --git a/crates/export/tests/opus_roundtrip.rs b/crates/export/tests/opus_roundtrip.rs new file mode 100644 index 000000000..5ea099f06 --- /dev/null +++ b/crates/export/tests/opus_roundtrip.rs @@ -0,0 +1,364 @@ +//! Round trip of `effectcraft-opusenc` through FilmCraft's spec-derived Opus decoder, plus an +//! ffmpeg oracle (skipped when ffmpeg is not installed) reading an Ogg Opus file written here. + +use std::process::Command; + +use effectcraft_opusenc::OpusEncoder; + +const SR: f32 = 48_000.0; +const FRAME: usize = OpusEncoder::FRAME_SIZE; + +/// Encodes interleaved `pcm`, padding the tail so the encoder's look-ahead is flushed. +fn encode(pcm: &[f32], channels: u8, bitrate: u32) -> (OpusEncoder, Vec>) { + let c = channels as usize; + let mut enc = OpusEncoder::new(channels, bitrate); + let total = pcm.len() / c + enc.pre_skip() as usize; + let frames = total.div_ceil(FRAME); + let mut padded = pcm.to_vec(); + padded.resize(frames * FRAME * c, 0.0); + let packets = padded.chunks(FRAME * c).map(|f| enc.encode_float(f)).collect(); + (enc, packets) +} + +/// Decodes with filmcraft-opus, drops the pre-skip and returns planar channels of `len` samples. +fn decode(head: &[u8], packets: &[Vec], len: usize) -> Vec> { + let mut dec = filmcraft_opus::Decoder::new(head).expect("OpusHead"); + dec.set_trim_pre_skip(true); + let mut out: Vec> = vec![Vec::new(); dec.channels()]; + for p in packets { + let planes = dec.decode(Some(p)).expect("decode"); + for (o, p) in out.iter_mut().zip(planes) { + o.extend_from_slice(&p); + } + } + for o in &mut out { + assert!(o.len() >= len, "decoded {} < {len}", o.len()); + o.truncate(len); + } + out +} + +fn roundtrip(planar: &[Vec], bitrate: u32) -> (Vec>, Vec>) { + let c = planar.len(); + let len = planar[0].len(); + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, c as u8, bitrate); + let decoded = decode(&enc.opus_head(), &packets, len); + (packets, decoded) +} + +/// SNR in dB over the steady part (the first and last 20 ms are excluded). +fn snr(orig: &[f32], dec: &[f32]) -> f64 { + let (a, b) = (FRAME, orig.len() - FRAME); + let mut s = 0f64; + let mut e = 0f64; + for i in a..b { + s += (orig[i] as f64).powi(2); + e += (orig[i] as f64 - dec[i] as f64).powi(2); + } + 10.0 * (s / e.max(1e-30)).log10() +} + +fn sine(freq: f32, amp: f32, len: usize) -> Vec { + (0..len).map(|i| amp * (2.0 * std::f32::consts::PI * freq * i as f32 / SR).sin()).collect() +} + +fn chirp(f0: f32, f1: f32, amp: f32, len: usize) -> Vec { + let dur = len as f32 / SR; + let k = (f1 - f0) / dur; + (0..len) + .map(|i| { + let t = i as f32 / SR; + amp * (2.0 * std::f32::consts::PI * (f0 * t + 0.5 * k * t * t)).sin() + }) + .collect() +} + +fn noise(amp: f32, len: usize, seed: u32) -> Vec { + let mut s = seed; + (0..len) + .map(|_| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + amp * ((s >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0) + }) + .collect() +} + +/// Energy (dB) of `x` in each of a few octave bands up to 16 kHz, from a Hann-windowed FFT +/// averaged over 2048-sample blocks. +fn band_energies(x: &[f32]) -> Vec { + const N: usize = 2048; + let edges = [100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0]; + let mut acc = vec![0f64; edges.len() - 1]; + let mut start = N; + while start + N <= x.len() - N { + let mut re: Vec = (0..N).map(|i| x[start + i] as f64 * (0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / N as f64).cos())).collect(); + let mut im = vec![0f64; N]; + fft(&mut re, &mut im); + for k in 1..N / 2 { + let f = k as f64 * SR as f64 / N as f64; + if let Some(b) = edges.windows(2).position(|w| f >= w[0] && f < w[1]) { + acc[b] += re[k] * re[k] + im[k] * im[k]; + } + } + start += N; + } + acc.iter().map(|e| 10.0 * (e + 1e-20).log10()).collect() +} + +fn fft(re: &mut [f64], im: &mut [f64]) { + let n = re.len(); + let mut j = 0; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let ang = -2.0 * std::f64::consts::PI / len as f64; + for s in (0..n).step_by(len) { + for k in 0..len / 2 { + let (wr, wi) = ((ang * k as f64).cos(), (ang * k as f64).sin()); + let (a, b) = (s + k, s + k + len / 2); + let tr = re[b] * wr - im[b] * wi; + let ti = re[b] * wi + im[b] * wr; + re[b] = re[a] - tr; + im[b] = im[a] - ti; + re[a] += tr; + im[a] += ti; + } + } + len <<= 1; + } +} + +#[test] +fn sine_440_mono() { + let x = sine(440.0, 0.5, 48_000); + for rate in [64_000, 128_000] { + let (packets, d) = roundtrip(std::slice::from_ref(&x), rate); + assert!(packets.iter().all(|p| p.len() == rate as usize / 400)); + let s = snr(&x, &d[0]); + eprintln!("sine 440 Hz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 20.0, "sine SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn chirp_mono() { + let x = chirp(100.0, 10_000.0, 0.4, 96_000); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("chirp 100 Hz-10 kHz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "chirp SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn white_noise_mono() { + let x = noise(0.3, 48_000, 12345); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + let (eo, ed) = (band_energies(&x), band_energies(&d[0])); + let err: Vec = x.iter().zip(&d[0]).map(|(a, b)| a - b).collect(); + let ee = band_energies(&err); + let octave_snr: Vec = eo.iter().zip(&ee).map(|(a, b)| a - b).collect(); + let worst = eo.iter().zip(&ed).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max); + let min_octave = octave_snr.iter().copied().fold(f64::MAX, f64::min); + eprintln!( + "white noise mono @ {} kb/s: full-band SNR {s:.1} dB, octave SNRs 100 Hz-16 kHz {:.1?} dB, worst octave energy error {worst:.2} dB", + rate / 1000, + octave_snr + ); + // Full-band noise loses everything above 20 kHz (CELT's last band ends there) and spends + // few bits per coefficient on 16-20 kHz, so the full-band waveform SNR is low; the band + // energies must match closely and the octaves below 16 kHz must be waveform-coded. + assert!(worst < 1.5, "band energy error {worst:.2} dB at {rate}: {eo:?} vs {ed:?}"); + assert!(s > 0.5, "noise SNR {s:.1} dB at {rate}"); + assert!(min_octave > if rate >= 128_000 { 10.0 } else { 3.0 }, "octave SNRs {octave_snr:?} at {rate}"); + } +} + +#[test] +fn lowpass_noise_mono() { + // White noise through a 4-pole low-pass at about 2 kHz: a dense spectrum the encoder can + // afford to code as a waveform. + let w = noise(0.8, 48_000, 999); + let a = (-2.0 * std::f32::consts::PI * 2_000.0 / SR).exp(); + let mut st = [0f32; 4]; + let x: Vec = w + .iter() + .map(|&v| { + let mut y = v; + for s in st.iter_mut() { + *s = (1.0 - a) * y + a * *s; + y = *s; + } + y + }) + .collect(); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("low-pass noise mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "low-pass noise SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn silence_stays_silent() { + let x = vec![0f32; 48_000]; + let (packets, d) = roundtrip(&[x.clone(), x], 128_000); + assert!(packets.iter().all(|p| p.len() == 3), "silence frames are 3-byte packets"); + for ch in &d { + let peak = ch.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(peak < 1e-6, "peak {peak}"); + } +} + +#[test] +fn stereo_distinct_channels() { + let l = sine(440.0, 0.5, 48_000); + let r = chirp(200.0, 6_000.0, 0.3, 48_000); + let (packets, d) = roundtrip(&[l.clone(), r.clone()], 256_000); + assert!(packets.iter().all(|p| p[0] == 0xFC && p.len() == 640)); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 256 kb/s: L (sine) SNR {sl:.1} dB, R (chirp) SNR {sr:.1} dB"); + assert!(sl > 20.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); + // No leakage of the chirp into the left channel (2-4 kHz octave: chirp only). + let (el, er) = (band_energies(&d[0]), band_energies(&r)); + assert!(el[4] < er[4] - 30.0, "chirp energy leaked into L: {} vs {}", el[4], er[4]); +} + +#[test] +fn stereo_128k_total() { + let l = sine(1000.0, 0.4, 48_000); + let r = sine(330.0, 0.4, 48_000); + let (_, d) = roundtrip(&[l.clone(), r.clone()], 128_000); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 128 kb/s: L SNR {sl:.1} dB, R SNR {sr:.1} dB"); + assert!(sl > 15.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); +} + +#[test] +fn signal_after_silence_and_loud_transient() { + // Silence, then a burst at full scale, then silence again: energy prediction must recover. + let mut x = vec![0f32; 24_000]; + x.extend(sine(2_000.0, 0.99, 24_000)); + x.extend(vec![0f32; 24_000]); + let (_, d) = roundtrip(std::slice::from_ref(&x), 96_000); + let s = snr(&x[24_000 + 2 * FRAME..48_000], &d[0][24_000 + 2 * FRAME..48_000]); + eprintln!("burst after silence @ 96 kb/s: SNR {s:.1} dB"); + assert!(s > 15.0, "burst SNR {s:.1}"); + let tail_peak = d[0][48_000 + 2 * FRAME..].iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(tail_peak < 1e-3, "tail peak {tail_peak}"); +} + +// ---- Ogg Opus (RFC 3533 + RFC 7845) for the ffmpeg oracle ---- + +fn ogg_crc(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + crc ^= (b as u32) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} + +fn ogg_page(out: &mut Vec, packet: &[u8], header_type: u8, granule: u64, seq: u32) { + let mut lacing = vec![255u8; packet.len() / 255]; + lacing.push((packet.len() % 255) as u8); + let start = out.len(); + out.extend_from_slice(b"OggS"); + out.push(0); + out.push(header_type); + out.extend_from_slice(&granule.to_le_bytes()); + out.extend_from_slice(&0x4543_4F50u32.to_le_bytes()); // serial + out.extend_from_slice(&seq.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); // CRC placeholder + out.push(lacing.len() as u8); + out.extend_from_slice(&lacing); + out.extend_from_slice(packet); + let crc = ogg_crc(&out[start..]); + out[start + 22..start + 26].copy_from_slice(&crc.to_le_bytes()); +} + +fn ogg_opus(enc: &OpusEncoder, packets: &[Vec], len: usize) -> Vec { + let mut out = Vec::new(); + ogg_page(&mut out, &enc.opus_head(), 0x02, 0, 0); + let vendor = b"effectcraft-opusenc"; + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes()); + tags.extend_from_slice(vendor); + tags.extend_from_slice(&0u32.to_le_bytes()); + ogg_page(&mut out, &tags, 0, 0, 1); + let pre_skip = enc.pre_skip() as u64; + for (i, p) in packets.iter().enumerate() { + let last = i + 1 == packets.len(); + let granule = if last { pre_skip + len as u64 } else { ((i + 1) * FRAME) as u64 }; + ogg_page(&mut out, p, if last { 0x04 } else { 0 }, granule, i as u32 + 2); + } + out +} + +#[test] +fn ogg_crc_known_value() { + // CRC-32 (poly 0x04C11DB7, no reflection, init 0) of "123456789". + assert_eq!(ogg_crc(b"123456789"), 0x89A1_897F); +} + +#[test] +fn ffmpeg_oracle_decodes_ogg_opus() { + if Command::new("ffmpeg").arg("-version").output().is_err() { + eprintln!("ffmpeg not installed; skipping the oracle test"); + return; + } + for (channels, rate) in [(1u8, 96_000u32), (2, 192_000)] { + let len = 48_000; + let planar: Vec> = + if channels == 1 { vec![chirp(150.0, 9_000.0, 0.4, len)] } else { vec![sine(440.0, 0.5, len), chirp(150.0, 9_000.0, 0.3, len)] }; + let c = channels as usize; + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, channels, rate); + let path = std::env::temp_dir().join(format!("effectcraft-opusenc-{}-{channels}.opus", std::process::id())); + std::fs::write(&path, ogg_opus(&enc, &packets, len)).unwrap(); + let out = Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "f32le", "-ac", &channels.to_string(), "-ar", "48000", "-"]) + .output() + .expect("run ffmpeg"); + let _ = std::fs::remove_file(&path); + assert!(out.status.success(), "ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr)); + let dec: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + assert_eq!(dec.len(), len * c, "ffmpeg must honour pre-skip and the end granule"); + for ch in 0..c { + let d: Vec = (0..len).map(|i| dec[i * c + ch]).collect(); + let s = snr(&planar[ch], &d); + eprintln!("ffmpeg {channels} ch @ {} kb/s, channel {ch}: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "ffmpeg-decoded SNR {s:.1} dB (channel {ch})"); + } + } +} diff --git a/crates/opusenc/Cargo.toml b/crates/opusenc/Cargo.toml new file mode 100644 index 000000000..90b0fd026 --- /dev/null +++ b/crates/opusenc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-opusenc" +description = "Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only, fullband 48 kHz, 20 ms; RFC 7845 OpusHead) for WebM export audio" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/opusenc/README.md b/crates/opusenc/README.md new file mode 100644 index 000000000..0c76d49f4 --- /dev/null +++ b/crates/opusenc/README.md @@ -0,0 +1,33 @@ +# effectcraft-opusenc + +Clean-room, pure-Rust Opus encoder for EffectCraft's WebM export audio (layer L0, `std` only, +no `unsafe`). + +- **CELT-only** packets (TOC configuration 31: fullband 20 kHz, 20 ms, one frame per packet), + 48 kHz input, mono or stereo (stereo coded as dual L/R stereo), constant packet size from the + requested bitrate; all-zero frames become 3-byte silence packets. +- Forward MDCT with the low-overlap window, band energies with inter/intra coarse prediction + (cheaper of the two per frame), fine and final energy bits, the decoder's exact bit allocation, + spreading and recursive band splitting, greedy PVQ search and codeword indexing. +- Not implemented: SILK/hybrid modes, transient (short-block) frames, TF changes, pitch + post-filter, intensity stereo, dynamic allocation boosts, VBR rate control. +- `OpusEncoder::pre_skip()` is 120 samples (the MDCT overlap); `opus_head()` returns the RFC 7845 + identification header. + +```rust +let mut enc = effectcraft_opusenc::OpusEncoder::new(2, 128_000); +let packet = enc.encode_float(&vec![0.0; effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2]); +``` + +## Specifications + +- RFC 6716, *Definition of the Opus Audio Codec* (September 2012), with the RFC 8251 updates + (October 2017) as implemented by the decoder we round-trip against. +- RFC 7845, *Ogg Encapsulation for the Opus Audio Codec* (April 2016), §5.1 `OpusHead` and + §4.2 pre-skip. + +Implemented from the RFC text. Constant tables (band edges, `eMeans`, the coarse-energy Laplace +model, the allocation table, ICDFs) and the decoder-side allocation arithmetic are mirrored from +FilmCraft's first-party, spec-derived `filmcraft-opus` decoder (MIT OR Apache-2.0), which is also +the round-trip oracle in `crates/export/tests/opus_roundtrip.rs` (ffmpeg is a second, external +oracle when installed). No third-party encoder or decoder source was consulted. diff --git a/crates/opusenc/src/bands.rs b/crates/opusenc/src/bands.rs new file mode 100644 index 000000000..744de3fe3 --- /dev/null +++ b/crates/opusenc/src/bands.rs @@ -0,0 +1,497 @@ +//! Band shape coding: PVQ search and codeword indexing, spreading, recursive band splitting +//! (RFC 6716 §4.3.4, encoder direction). +//! +//! The encoder codes 20 ms long-block frames only (no transients, no TF changes), and stereo +//! as dual (independent L/R) stereo, so a band is always coded by `quant_partition` with a single +//! block. Every bit-budget computation mirrors the decoder so both sides take the same +//! split/pulse decisions. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::rate::{Mode, QTHETA_OFFSET, get_pulses}; +use crate::tables::*; + +pub const SPREAD_NORMAL: usize = 2; + +#[inline] +fn frac_mul16(a: i32, b: i32) -> i32 { + (16384 + (a as i16 as i32) * (b as i16 as i32)) >> 15 +} + +fn bitexact_cos(x: i32) -> i32 { + let tmp = (4096 + x * x) >> 13; + let x2 = tmp; + let x2 = (32767 - x2) + frac_mul16(x2, -7651 + frac_mul16(x2, 8277 + frac_mul16(-626, x2))); + 1 + x2 +} + +fn bitexact_log2tan(isin: i32, icos: i32) -> i32 { + let lc = ilog(icos as u32); + let ls = ilog(isin as u32); + let icos = icos << (15 - lc); + let isin = isin << (15 - ls); + (ls - lc) * (1 << 11) + frac_mul16(isin, frac_mul16(isin, -2597) + 7932) - frac_mul16(icos, frac_mul16(icos, -2597) + 7932) +} + +fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 { + const EXP2_TABLE8: [i32; 8] = [16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048]; + let mut n2 = 2 * n - 1; + if stereo && n == 2 { + n2 -= 1; + } + let mut qb = (b + n2 * offset) / n2; + qb = qb.min(b - pulse_cap - (4 << BITRES)); + qb = qb.min(8 << BITRES); + if qb < (1 << BITRES >> 1) { + 1 + } else { + let qn = EXP2_TABLE8[(qb & 0x7) as usize] >> (14 - (qb >> BITRES)); + (qn + 1) >> 1 << 1 + } +} + +/// One elementary pass of the spreading rotation, inverted (the decoder applies the forward +/// pass; the encoder undoes it before the pulse search). +fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + if len <= stride { + return; + } + if len >= 2 * stride + 1 { + for i in 0..=len - 2 * stride - 1 { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } + } + for i in (0..len - stride).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } +} + +/// Inverse of the decoder's spreading rotation (§4.3.4.3) for one block of `len` samples. +pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + // The decoder applies (stride2: s, c) then (1: c, s); undo in reverse order. + exp_rotation1_inv(x, len, 1, c, s); + if stride2 != 0 { + exp_rotation1_inv(x, len, stride2, s, c); + } +} + +/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`. +pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { + let n = x.len(); + y[..n].fill(0); + let abs: Vec = x.iter().map(|v| v.abs()).collect(); + let sum: f32 = abs.iter().sum(); + if !sum.is_finite() || sum <= 1e-15 { + y[0] = k as i32; + return; + } + let mut placed = 0usize; + let mut xy = 0f32; + let mut yy = 0f32; + if k > n / 2 { + // Pre-project onto the pyramid (rounding down) and place the rest greedily. + let r = (k as f32 - 1.0) / sum; + for j in 0..n { + let p = (abs[j] * r).floor() as i32; + y[j] = p; + placed += p as usize; + xy += abs[j] * p as f32; + yy += (p * p) as f32; + } + } + while placed < k { + let mut best = 0usize; + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } + xy += abs[best]; + yy += 2.0 * y[best] as f32 + 1.0; + y[best] += 1; + placed += 1; + } + for j in 0..n { + if x[j] < 0.0 { + y[j] = -y[j]; + } + } +} + +/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps +/// back to `y`. +pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec) { + let n = y.len(); + let w = k + 1; + v.clear(); + v.resize((n + 1) * w, 0); + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + let a = v[(nn - 1) * w + kk]; + let b = v[nn * w + kk - 1]; + let c = v[(nn - 1) * w + kk - 1]; + v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + } + } + let total = v[n * w + k]; + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut index = 0u64; + let mut kk = k; + for j in 0..n { + if kk == 0 { + break; + } + let rem = n - j; + let t = y[j].unsigned_abs() as usize; + // Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half. + let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum(); + if y[j] < 0 { + base += (vv(rem - 1, kk) + vv(rem, kk)) / 2; + } + index += base; + kk -= t; + } + debug_assert!(total <= u32::MAX as u64); + if total >= 2 { + enc.uint(index as u32, total as u32); + } +} + +pub struct BandEnc<'m, 'e> { + pub m: &'m Mode, + pub enc: &'e mut RangeEncoder, + pub i: usize, + pub spread: usize, + pub remaining_bits: i32, + pvq_v: Vec, + iy: Vec, + tmp: Vec, +} + +impl BandEnc<'_, '_> { + /// `compute_theta` for a mono split (single block): measures the energy split between the + /// two halves and codes the quantised angle with the triangular pdf. + fn compute_theta_split(&mut self, xa: &[f32], xb: &[f32], b: &mut i32, lm: i32) -> (i32, i32, i32) { + let n = xa.len() as i32; + let i = self.i; + let pulse_cap = self.m.log_n[i] + lm * (1 << BITRES); + let offset = (pulse_cap >> 1) - QTHETA_OFFSET; + let qn = compute_qn(n, *b, offset, pulse_cap, false); + let tell = self.enc.tell_frac(); + let mut itheta = 0i32; + if qn != 1 { + let mid = xa.iter().map(|v| v * v).sum::().sqrt(); + let side = xb.iter().map(|v| v * v).sum::().sqrt(); + let ang = (side as f64).atan2(mid as f64) * std::f64::consts::FRAC_2_PI; + let it = ((ang * qn as f64 + 0.5).floor() as i32).clamp(0, qn) as u32; + let q = qn as u32; + let half = q >> 1; + let ft = (half + 1) * (half + 1); + let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) }; + self.enc.encode(fl, fl + fs, ft); + itheta = ((it * 16384) / q) as i32; + } + let qalloc = self.enc.tell_frac() - tell; + *b -= qalloc; + let delta = if itheta == 0 { + -16384 + } else if itheta == 16384 { + 16384 + } else { + let imid = bitexact_cos(itheta); + let iside = bitexact_cos(16384 - itheta); + frac_mul16((n - 1) << 7, bitexact_log2tan(iside, imid)) + }; + (itheta, delta, qalloc) + } + + /// `quant_partition` (single block, encoder side). + pub fn quant_partition(&mut self, x: &[f32], mut b: i32, mut lm: i32) { + let n = x.len(); + let i = self.i; + let cache = self.m.cache(lm, i); + let cache_max = cache[cache[0] as usize] as i32; + if lm != -1 && b > cache_max + 12 && n > 2 { + let half = n >> 1; + let (xa, xb) = x.split_at(half); + lm -= 1; + let (itheta, delta, qalloc) = self.compute_theta_split(xa, xb, &mut b, lm); + let mut mbits = 0.max(b.min((b - delta) / 2)); + let mut sbits = b - mbits; + self.remaining_bits -= qalloc; + let mut rebalance = self.remaining_bits; + if mbits >= sbits { + self.quant_partition(xa, mbits, lm); + rebalance = mbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 0 { + sbits += rebalance - (3 << BITRES); + } + self.quant_partition(xb, sbits, lm); + } else { + self.quant_partition(xb, sbits, lm); + rebalance = sbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 16384 { + mbits += rebalance - (3 << BITRES); + } + self.quant_partition(xa, mbits, lm); + } + return; + } + let m = self.m; + let mut q = m.bits2pulses(i, lm, b); + let mut curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + while self.remaining_bits < 0 && q > 0 { + self.remaining_bits += curr_bits; + q -= 1; + curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + } + if q != 0 { + let k = get_pulses(q) as usize; + let mut tmp = std::mem::take(&mut self.tmp); + tmp.clear(); + tmp.extend_from_slice(x); + exp_rotation_inv(&mut tmp, n, k, self.spread); + let mut iy = std::mem::take(&mut self.iy); + iy.clear(); + iy.resize(n, 0); + pvq_search(&tmp, k, &mut iy); + encode_pulses(&iy, k, self.enc, &mut self.pvq_v); + self.iy = iy; + self.tmp = tmp; + } + } +} + +/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels +/// of `n` normalised MDCT coefficients each. +#[allow(clippy::too_many_arguments)] +pub fn quant_all_bands( + m: &Mode, + start: usize, + end: usize, + x: &[f32], + n_total: usize, + c: usize, + pulses: &[i32; NB_EBANDS], + spread: usize, + dual_stereo: bool, + intensity: usize, + total_bits: i32, + mut balance: i32, + enc: &mut RangeEncoder, + lm: usize, + coded_bands: usize, +) { + let mm = 1usize << lm; + debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo"); + let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() }; + for i in start..end { + ctx.i = i; + let xo = mm * EBANDS[i] as usize; + let n = mm * EBANDS[i + 1] as usize - xo; + let tell = ctx.enc.tell_frac(); + if i != start { + balance -= tell; + } + let remaining_bits = total_bits - tell - 1; + ctx.remaining_bits = remaining_bits; + let b = if i < coded_bands { + let curr_balance = balance / 3.min(coded_bands as i32 - i as i32); + 0.max(16383.min((remaining_bits + 1).min(pulses[i] + curr_balance))) + } else { + 0 + }; + // Bands at and above `intensity` (= coded_bands) carry no bits, so they emit no symbols. + if i < coded_bands || c == 1 { + if c == 2 && dual_stereo && i < intensity { + ctx.quant_partition(&x[xo..xo + n], b / 2, lm as i32); + ctx.quant_partition(&x[n_total + xo..n_total + xo + n], b / 2, lm as i32); + } else { + ctx.quant_partition(&x[xo..xo + n], b, lm as i32); + } + } + balance += pulses[i] + tell; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + fn exp_rotation1(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + let ms = -s; + if len <= stride { + return; + } + for i in 0..len - stride { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + if len >= 2 * stride + 1 { + for i in (0..=len - 2 * stride - 1).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + } + } + + /// The decoder's spreading rotation (single block). + fn exp_rotation(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + if stride2 != 0 { + exp_rotation1(x, len, stride2, s, c); + } + exp_rotation1(x, len, 1, c, s); + } + + #[test] + fn spreading_inverse_is_exact() { + for &len in &[8usize, 16, 24, 48, 96, 176] { + for k in [1usize, 2, 3, 5] { + let orig: Vec = (0..len).map(|i| ((i * 31 % 17) as f32 - 8.0) / 8.0).collect(); + let mut x = orig.clone(); + exp_rotation_inv(&mut x, len, k, SPREAD_NORMAL); + exp_rotation(&mut x, len, k, SPREAD_NORMAL); + for (a, b) in x.iter().zip(&orig) { + assert!((a - b).abs() < 1e-4, "len={len} k={k}"); + } + } + } + } + + /// PVQ decoder of RFC 6716 §4.3.4.2 (as in FilmCraft's decoder). + fn decode_pulses(n: usize, k: usize, dec: &mut RangeDecoder) -> Vec { + let w = k + 1; + let mut v = vec![0u64; (n + 1) * w]; + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + v[nn * w + kk] = v[(nn - 1) * w + kk].saturating_add(v[nn * w + kk - 1]).saturating_add(v[(nn - 1) * w + kk - 1]); + } + } + let total = v[n * w + k]; + let mut i = if total >= 2 { dec.uint(total as u32) as u64 } else { 0 }; + let mut y = vec![0i32; n]; + let mut kk = k; + for j in 0..n { + let rem = n - j; + if kk == 0 { + break; + } + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut p = (vv(rem - 1, kk) + vv(rem, kk)) / 2; + let neg = i >= p; + if neg { + i -= p; + } + let k0 = kk; + p -= vv(rem - 1, kk); + while p > i && kk > 0 { + kk -= 1; + p -= vv(rem - 1, kk); + } + let mag = (k0 - kk) as i32; + y[j] = if neg { -mag } else { mag }; + i -= p.min(i); + } + y + } + + fn all_codewords(n: usize, k: usize) -> Vec> { + if n == 0 { + return if k == 0 { vec![vec![]] } else { vec![] }; + } + let mut out = Vec::new(); + for t in -(k as i32)..=(k as i32) { + for mut rest in all_codewords(n - 1, k - t.unsigned_abs() as usize) { + rest.insert(0, t); + out.push(rest); + } + } + out + } + + #[test] + fn pvq_index_roundtrip_exhaustive() { + let mut v = Vec::new(); + for n in 1..6usize { + for k in 1..6usize { + for y in all_codewords(n, k) { + let mut e = RangeEncoder::new(); + encode_pulses(&y, k, &mut e, &mut v); + let buf = e.done(16); + let mut d = RangeDecoder::new(&buf); + assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}"); + } + } + } + } + + #[test] + fn pvq_search_finds_good_direction() { + let x: Vec = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect(); + let mut y = vec![0i32; 32]; + for k in [1usize, 4, 16, 64] { + pvq_search(&x, k, &mut y); + assert_eq!(y.iter().map(|v| v.unsigned_abs() as usize).sum::(), k); + let xy: f32 = x.iter().zip(&y).map(|(a, &b)| a * b as f32).sum(); + let yy: f32 = y.iter().map(|&b| (b * b) as f32).sum(); + let xx: f32 = x.iter().map(|a| a * a).sum(); + let corr = xy / (xx * yy).sqrt(); + assert!(corr > if k >= 16 { 0.9 } else { 0.2 }, "k={k} corr={corr}"); + } + } +} diff --git a/crates/opusenc/src/energy.rs b/crates/opusenc/src/energy.rs new file mode 100644 index 000000000..0057ae33a --- /dev/null +++ b/crates/opusenc/src/energy.rs @@ -0,0 +1,230 @@ +//! Band energy quantisation: Laplace-coded coarse energy with inter/intra prediction, fine +//! energy and the final refinement bits (RFC 6716 §4.3.2, encoder direction). +//! +//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction +//! state never drifts between encoder and decoder. + +use crate::range::RangeEncoder; +use crate::rate::MAX_FINE_BITS; +use crate::tables::*; + +const LAPLACE_MINP: u32 = 1; +const LAPLACE_NMIN: u32 = 16; + +fn laplace_get_freq1(fs0: u32, decay: u32) -> u32 { + let ft = 32768 - LAPLACE_MINP * (2 * LAPLACE_NMIN) - fs0; + (ft * (16384 - decay)) >> 15 +} + +/// Encodes a Laplace-distributed integer (`fs` = probability of 0 in Q15, `decay` in Q14). +/// Returns the value actually coded (very large magnitudes are clamped to the representable +/// range). +pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u32) -> i32 { + let mut fl = 0u32; + let mut coded = value; + if value != 0 { + let s: i32 = if value < 0 { -1 } else { 0 }; + let val = value.abs(); + fl = fs; + fs = laplace_get_freq1(fs, decay); + let mut i = 1; + while fs > 0 && i < val { + fs *= 2; + fl += fs + 2 * LAPLACE_MINP; + fs = (fs * decay) >> 15; + i += 1; + } + if fs == 0 { + let ndi_max = (32768 - fl + LAPLACE_MINP - 1) as i32; + let ndi_max = (ndi_max - s) >> 1; + let di = (val - i).min(ndi_max - 1); + fl = (fl as i32 + (2 * di + 1 + s) * LAPLACE_MINP as i32) as u32; + fs = LAPLACE_MINP.min(32768 - fl); + let mag = i + di; + coded = if s < 0 { -mag } else { mag }; + } else { + fs += LAPLACE_MINP; + if s == 0 { + fl += fs; + } + } + } + enc.encode_bin(fl, fl + fs, 15); + coded +} + +/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative +/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in +/// bits. +#[allow(clippy::too_many_arguments)] +pub fn quant_coarse_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + intra: bool, + enc: &mut RangeEncoder, + c: usize, + lm: usize, + budget: i32, +) { + let prob = &E_PROB_MODEL[lm][intra as usize]; + let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) }; + let mut prev = [0f32; 2]; + for i in start..end { + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let oldc = old_e[idx].max(-9.0); + let x = e[idx]; + let f = x - coef * oldc - prev[ch]; + let mut qi = (f + 0.5).floor().clamp(-64.0, 64.0) as i32; + let tell = enc.tell(); + // Keep a few bits per remaining band in reserve when the budget runs out. + let bits_left = budget - tell - 3 * c as i32 * (end - i) as i32; + if i != start && bits_left < 30 { + if bits_left < 24 { + qi = qi.min(1); + } + if bits_left < 16 { + qi = qi.max(-1); + } + } + if budget - tell >= 15 { + let pi = 2 * i.min(20); + qi = laplace_encode(enc, qi, (prob[pi] as u32) << 7, (prob[pi + 1] as u32) << 6); + } else if budget - tell >= 2 { + qi = qi.clamp(-1, 1); + enc.icdf(if qi < 0 { (-2 * qi - 1) as usize } else { (2 * qi) as usize }, &SMALL_ENERGY_ICDF, 2); + } else if budget - tell >= 1 { + qi = qi.clamp(-1, 0); + enc.bit_logp(qi == -1, 1); + } else { + qi = -1; + } + let q = qi as f32; + old_e[idx] = oldc; + let tmp = coef * old_e[idx] + prev[ch] + q; + old_e[idx] = tmp; + prev[ch] = prev[ch] + q - beta * q; + } + } +} + +/// `quant_fine_energy`: raw bits refining each band's quantised energy. +pub fn quant_fine_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + enc: &mut RangeEncoder, + c: usize, +) { + for i in start..end { + let fq = fine_quant[i]; + if fq <= 0 { + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let err = e[idx] - old_e[idx]; + let levels = 1i32 << fq; + let q2 = (((err + 0.5) * levels as f32).floor() as i32).clamp(0, levels - 1); + enc.bits(q2 as u32, fq as u32); + let offset = (q2 as f32 + 0.5) * (1 << (14 - fq)) as f32 / 16384.0 - 0.5; + old_e[idx] += offset; + } + } +} + +/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of +/// energy resolution per band, in priority order. +#[allow(clippy::too_many_arguments)] +pub fn quant_energy_finalise( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + fine_priority: &[i32; NB_EBANDS], + mut bits_left: i32, + enc: &mut RangeEncoder, + c: usize, +) { + for prio in 0..2 { + let mut i = start; + while i < end && bits_left >= c as i32 { + if fine_quant[i] >= MAX_FINE_BITS || fine_priority[i] != prio { + i += 1; + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let q2 = (e[idx] - old_e[idx] >= 0.0) as u32; + enc.bits(q2, 1); + let offset = (q2 as f32 - 0.5) * (1 << (14 - fine_quant[i] - 1)) as f32 / 16384.0; + old_e[idx] += offset; + bits_left -= 1; + } + i += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + /// Laplace decoder of RFC 6716 §4.3.2.1 (as in FilmCraft's decoder). + fn laplace_decode(dec: &mut RangeDecoder, mut fs: u32, decay: u32) -> i32 { + let mut val = 0i32; + let fm = dec.decode(32768); + let mut fl = 0u32; + if fm >= fs { + val += 1; + fl = fs; + fs = laplace_get_freq1(fs, decay) + LAPLACE_MINP; + while fs > LAPLACE_MINP && fm >= fl + 2 * fs { + fs *= 2; + fl += fs; + fs = (((fs - 2 * LAPLACE_MINP) * decay) >> 15) + LAPLACE_MINP; + val += 1; + } + if fs <= LAPLACE_MINP { + let di = (fm - fl) >> 1; + val += di as i32; + fl += 2 * di * LAPLACE_MINP; + } + if fm < fl + fs { + val = -val; + } else { + fl += fs; + } + } + dec.update(fl, (fl + fs).min(32768), 32768); + val + } + + #[test] + fn laplace_roundtrip_all_models() { + for lm in 0..4 { + for intra in 0..2 { + for band in 0..21 { + let prob = &E_PROB_MODEL[lm][intra]; + let (fs, decay) = ((prob[2 * band] as u32) << 7, (prob[2 * band + 1] as u32) << 6); + let vals: Vec = (-64..=64).collect(); + let mut e = RangeEncoder::new(); + let coded: Vec = vals.iter().map(|&v| laplace_encode(&mut e, v, fs, decay)).collect(); + let size = (e.tell() as usize).div_ceil(8); + let buf = e.done(size); + let mut d = RangeDecoder::new(&buf); + for (&v, &c) in vals.iter().zip(&coded) { + assert!(c == v || (c.signum() == v.signum() && c.abs() < v.abs()), "clamp {v} -> {c}"); + assert_eq!(laplace_decode(&mut d, fs, decay), c, "lm={lm} intra={intra} band={band} v={v}"); + } + } + } + } + } +} diff --git a/crates/opusenc/src/lib.rs b/crates/opusenc/src/lib.rs new file mode 100644 index 000000000..1e7a2cec8 --- /dev/null +++ b/crates/opusenc/src/lib.rs @@ -0,0 +1,311 @@ +//! Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only mode; RFC 7845 `OpusHead`). +//! +//! Produces fullband (20 kHz) 48 kHz Opus packets of 20 ms, mono or stereo, at a constant +//! packet size derived from the requested bitrate. Each packet is a single CELT frame (TOC +//! configuration 31, frame-count code 0). Stereo is coded as dual (independent L/R) stereo. +//! +//! Layer L0: no dependencies beyond `std`; no `unsafe`; builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use effectcraft_opusenc::OpusEncoder; +//! let mut enc = OpusEncoder::new(2, 128_000); +//! let pcm = vec![0.0f32; OpusEncoder::FRAME_SIZE * 2]; +//! let packet = enc.encode_float(&pcm); +//! assert_eq!(packet[0] >> 3, 31); // CELT-only, fullband, 20 ms +//! assert_eq!(enc.opus_head().len(), 19); +//! ``` + +// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions. +#![allow(clippy::precedence, clippy::int_plus_one)] + +mod bands; +mod energy; +mod mdct; +mod range; +mod rate; +mod tables; + +use std::sync::OnceLock; + +use bands::{SPREAD_NORMAL, quant_all_bands}; +use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy}; +use mdct::Mdct; +use range::{BITRES, RangeEncoder}; +use rate::{Mode, compute_allocation}; +use tables::*; + +/// Samples per channel in one 20 ms frame at 48 kHz. +const N: usize = 960; +/// LM of a 20 ms frame (8 short blocks). +const LM: usize = 3; +/// Smallest CELT payload the encoder produces (bytes). +const MIN_FRAME_BYTES: usize = 16; +/// Largest CELT payload allowed in one frame (RFC 6716 §3.4, R2). +const MAX_FRAME_BYTES: usize = 1275; + +fn mode() -> &'static Mode { + static MODE: OnceLock = OnceLock::new(); + MODE.get_or_init(Mode::new) +} + +/// Opus encoder for one mono or stereo stream at 48 kHz. +pub struct OpusEncoder { + channels: usize, + /// CELT payload bytes per packet (the packet is one byte longer: the TOC). + frame_bytes: usize, + mdct: Mdct, + /// Last input sample of each channel (pre-emphasis filter memory, ×32768). + preemph_mem: [f32; 2], + /// Last `OVERLAP` pre-emphasised samples of each channel. + hist: Vec, + /// Quantised band energies, exactly as the decoder holds them. + old_band_e: [f32; 2 * NB_EBANDS], + block: Vec, + freq: Vec, +} + +impl OpusEncoder { + /// Samples per channel per packet (20 ms at 48 kHz). + pub const FRAME_SIZE: usize = N; + + /// Creates an encoder for `channels` (1 or 2) channels of 48 kHz audio at `bitrate_bps` + /// (total for all channels; packets are constant-size, clamped to 6.8–510 kb/s). + pub fn new(channels: u8, bitrate_bps: u32) -> Self { + assert!(channels == 1 || channels == 2, "Opus family-0 streams have 1 or 2 channels"); + let c = channels as usize; + let packet_bytes = (bitrate_bps as usize).div_ceil(400); + let frame_bytes = packet_bytes.saturating_sub(1).clamp(MIN_FRAME_BYTES, MAX_FRAME_BYTES); + OpusEncoder { + channels: c, + frame_bytes, + mdct: Mdct::new(N), + preemph_mem: [0.0; 2], + hist: vec![0.0; c * OVERLAP], + old_band_e: [-28.0; 2 * NB_EBANDS], + block: vec![0.0; N + OVERLAP], + freq: vec![0.0; c * N], + } + } + + /// Number of channels. + pub fn channels(&self) -> u8 { + self.channels as u8 + } + + /// Actual bitrate in bits per second (packet size × 50 packets per second). + pub fn bitrate(&self) -> u32 { + ((self.frame_bytes + 1) * 8 * 50) as u32 + } + + /// Samples at 48 kHz the decoder must discard from the start of the stream (the MDCT + /// overlap); this is the `OpusHead` pre-skip. + pub fn pre_skip(&self) -> u16 { + OVERLAP as u16 + } + + /// The 19-byte `OpusHead` identification header (RFC 7845 §5.1): version 1, channel count, + /// pre-skip, input sample rate 48000, output gain 0, channel mapping family 0. + pub fn opus_head(&self) -> Vec { + let mut h = Vec::with_capacity(19); + h.extend_from_slice(b"OpusHead"); + h.push(1); + h.push(self.channels as u8); + h.extend_from_slice(&self.pre_skip().to_le_bytes()); + h.extend_from_slice(&48_000u32.to_le_bytes()); + h.extend_from_slice(&0i16.to_le_bytes()); + h.push(0); + h + } + + /// Encodes one 20 ms frame of interleaved f32 samples in [-1, 1] (exactly + /// `FRAME_SIZE * channels` values) and returns one Opus packet (TOC byte + CELT frame). + pub fn encode_float(&mut self, pcm: &[f32]) -> Vec { + let c = self.channels; + assert_eq!(pcm.len(), N * c, "encode_float takes exactly FRAME_SIZE samples per channel"); + + // Pre-emphasis, the MDCT and the band energies. + let mut band_e = [0f32; 2 * NB_EBANDS]; + let mut log_e = [0f32; 2 * NB_EBANDS]; + let mut peak = 0f32; + for ch in 0..c { + self.block[..OVERLAP].copy_from_slice(&self.hist[ch * OVERLAP..(ch + 1) * OVERLAP]); + let mut mem = self.preemph_mem[ch]; + for j in 0..N { + let s = pcm[j * c + ch]; + let s = if s.is_finite() { s.clamp(-2.0, 2.0) * 32768.0 } else { 0.0 }; + let v = s - PREEMPH * mem; + mem = s; + self.block[OVERLAP + j] = v; + } + self.preemph_mem[ch] = mem; + self.hist[ch * OVERLAP..(ch + 1) * OVERLAP].copy_from_slice(&self.block[N..]); + peak = self.block.iter().fold(peak, |p, v| p.max(v.abs())); + let freq = &mut self.freq[ch * N..(ch + 1) * N]; + self.mdct.forward(&self.block, freq); + for i in 0..NB_EBANDS { + let lo = (EBANDS[i] as usize) << LM; + let hi = (EBANDS[i + 1] as usize) << LM; + let e = (1e-27f32 + freq[lo..hi].iter().map(|v| v * v).sum::()).sqrt(); + band_e[ch * NB_EBANDS + i] = e; + log_e[ch * NB_EBANDS + i] = e.log2() - E_MEANS[i]; + let g = 1.0 / e; + for v in freq[lo..hi].iter_mut() { + *v *= g; + } + } + } + let silence = peak < 1e-4; + let toc = (31u8 << 3) | if c == 2 { 4 } else { 0 }; + let mut packet = vec![toc]; + packet.extend_from_slice(&self.encode_celt(&log_e, silence)); + packet + } + + /// Codes one CELT frame (RFC 6716 §4.3, in bitstream order). + fn encode_celt(&mut self, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec { + let m = mode(); + let c = self.channels; + let (start, end) = (0usize, NB_EBANDS); + let mut enc = RangeEncoder::new(); + if c == 1 { + for i in 0..NB_EBANDS { + self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]); + } + } + if silence { + // The silence flag; the decoder treats every remaining bit as consumed. + enc.bit_logp(true, 15); + self.old_band_e = [-28.0; 2 * NB_EBANDS]; + return enc.done(2); + } + let len = self.frame_bytes; + let total_bits_raw = len as i32 * 8; + enc.bit_logp(false, 15); + // No pitch post-filter. + if enc.tell() + 16 <= total_bits_raw { + enc.bit_logp(false, 1); + } + // Long blocks only. + if enc.tell() + 3 <= total_bits_raw { + enc.bit_logp(false, 3); + } + // Coarse energy, choosing intra or inter prediction by trial (whichever is cheaper). + if enc.tell() + 3 <= total_bits_raw { + let mut enc_intra = enc.clone(); + let mut old_intra = self.old_band_e; + enc_intra.bit_logp(true, 3); + quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw); + enc.bit_logp(false, 3); + let mut old_inter = self.old_band_e; + quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw); + if enc_intra.tell_frac() < enc.tell_frac() { + enc = enc_intra; + self.old_band_e = old_intra; + } else { + self.old_band_e = old_inter; + } + } else { + quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw); + } + // TF resolution: no changes. + { + let mut budget = total_bits_raw; + let mut tell = enc.tell(); + let mut logp = 4; + let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget; + budget -= tf_select_rsv as i32; + for _ in start..end { + if tell + logp <= budget { + enc.bit_logp(false, logp as u32); + tell = enc.tell(); + } + logp = 5; + } + if tf_select_rsv && TF_SELECT_TABLE[LM][0] != TF_SELECT_TABLE[LM][2] { + enc.bit_logp(false, 1); + } + } + if enc.tell() + 4 <= total_bits_raw { + enc.icdf(SPREAD_NORMAL, &SPREAD_ICDF, 5); + } + // Dynamic allocation: no boosts. + let cap = m.init_caps(LM, c); + let offsets = [0i32; NB_EBANDS]; + let total_bits = total_bits_raw << BITRES; + let mut tellf = enc.tell_frac(); + for &cp in cap.iter().take(end).skip(start) { + if tellf + (6 << BITRES) < total_bits && 0 < cp { + enc.bit_logp(false, 6); + tellf = enc.tell_frac(); + } + } + let alloc_trim = 5; + if tellf + (6 << BITRES) <= total_bits { + enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7); + } + let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1; + let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc); + quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c); + quant_all_bands( + m, + start, + end, + &self.freq, + N, + c, + &alloc.pulses, + SPREAD_NORMAL, + alloc.dual_stereo, + alloc.intensity, + total_bits_raw << BITRES, + alloc.balance, + &mut enc, + LM, + alloc.coded_bands, + ); + let bits_left = total_bits_raw - enc.tell(); + quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c); + debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw); + if c == 1 { + let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS); + b.copy_from_slice(a); + } + enc.done(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opus_head_layout() { + let e = OpusEncoder::new(2, 96_000); + let h = e.opus_head(); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(h[8], 1); + assert_eq!(h[9], 2); + assert_eq!(u16::from_le_bytes([h[10], h[11]]), e.pre_skip()); + assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000); + assert_eq!(&h[16..], &[0, 0, 0]); + } + + #[test] + fn packet_sizes_follow_bitrate() { + for (ch, rate) in [(1u8, 64_000u32), (2, 128_000), (2, 256_000)] { + let mut e = OpusEncoder::new(ch, rate); + let pcm: Vec = (0..N * ch as usize).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); + let p = e.encode_float(&pcm); + assert_eq!(p.len(), rate as usize / 400); + assert_eq!(p[0], 0xF8 | if ch == 2 { 4 } else { 0 }); + assert_eq!(e.bitrate(), rate); + } + } + + #[test] + fn silence_is_a_tiny_packet() { + let mut e = OpusEncoder::new(1, 64_000); + let p = e.encode_float(&vec![0.0; N]); + assert_eq!(p.len(), 3); + } +} diff --git a/crates/opusenc/src/mdct.rs b/crates/opusenc/src/mdct.rs new file mode 100644 index 000000000..814de0375 --- /dev/null +++ b/crates/opusenc/src/mdct.rs @@ -0,0 +1,293 @@ +//! Forward MDCT with the CELT low-overlap window (RFC 6716 §4.3.7, run in the analysis +//! direction). +//! +//! A 20 ms frame is a 1920-point MDCT (960 coefficients) whose window is zero for 420 samples, +//! rises over the 120-sample overlap, is flat for 840 samples, falls over the next 120 samples +//! and is zero again for the last 420. The input block is therefore the 120 overlap samples kept +//! from the previous frame followed by the 960 new samples. The MDCT is folded into a DCT-IV, +//! which is computed with an N/2-point complex FFT. + +use crate::tables::OVERLAP; + +#[derive(Clone, Copy, Default, Debug)] +pub struct Cpx { + pub re: f32, + pub im: f32, +} + +impl Cpx { + #[inline] + fn mul(self, o: Cpx) -> Cpx { + Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re } + } + #[inline] + fn add(self, o: Cpx) -> Cpx { + Cpx { re: self.re + o.re, im: self.im + o.im } + } +} + +/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive). +pub struct Fft { + n: usize, + factors: Vec, + twiddle: Vec, +} + +impl Fft { + pub fn new(n: usize) -> Fft { + let mut factors = Vec::new(); + let mut m = n; + for p in [4usize, 2, 3, 5] { + while m.is_multiple_of(p) { + factors.push(p); + m /= p; + } + } + assert_eq!(m, 1, "unsupported FFT size {n}"); + let twiddle = (0..n) + .map(|k| { + let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + Fft { n, factors, twiddle } + } + + pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec) { + scratch.clear(); + scratch.extend_from_slice(&data[..self.n]); + self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1); + } + + fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) { + let p = self.factors[fi]; + let m = n / p; + if m == 1 { + for (i, o) in out.iter_mut().enumerate().take(p) { + *o = inp[i * in_stride]; + } + } else { + for q in 0..p { + self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p); + } + } + let big = self.n; + let mut t = [Cpx::default(); 5]; + for k in 0..m { + for (q, tq) in t.iter_mut().enumerate().take(p) { + *tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]); + } + for q2 in 0..p { + let mut acc = Cpx::default(); + for (q, tq) in t.iter().enumerate().take(p) { + acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)])); + } + out[q2 * m + k] = acc; + } + } + } +} + +/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples. +pub struct Mdct { + /// Number of coefficients (half the MDCT length). + n: usize, + fft: Fft, + pre: Vec, + post: Vec, + /// Rising half of the low-overlap window (RFC 6716 §4.3.7). + pub window: [f32; OVERLAP], + scale: f32, + z: Vec, + scratch: Vec, + v: Vec, +} + +impl Mdct { + pub fn new(n: usize) -> Mdct { + let n2 = n / 2; + let pi = std::f64::consts::PI; + let pre = (0..n2) + .map(|m| { + let ph = -pi * m as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let post = (0..n2) + .map(|p| { + let ph = -pi * (p as f64 + 0.25) / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let mut window = [0f32; OVERLAP]; + for (i, w) in window.iter_mut().enumerate() { + let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64; + let s = x.sin(); + *w = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32; + } + Mdct { n, fft: Fft::new(n2), pre, post, window, scale: 2.0 / n as f32, z: Vec::new(), scratch: Vec::new(), v: vec![0.0; n] } + } + + /// Transforms `block` (`n + OVERLAP` samples: the previous frame's last `OVERLAP` samples, then + /// `n` new ones) into `out[..n]`, scaled so that the RFC 6716 decoder's inverse MDCT and + /// overlap-add reproduce the input. + pub fn forward(&mut self, block: &[f32], out: &mut [f32]) { + let n = self.n; + let n2 = n / 2; + debug_assert_eq!(block.len(), n + OVERLAP); + // Place the windowed block into the (virtual) 2n-sample MDCT input u at offset `pad`. + let pad = (n - OVERLAP) / 2; + let w = &self.window; + let u = |t: usize| -> f32 { + if t < pad || t >= pad + n + OVERLAP { + return 0.0; + } + let j = t - pad; + let g = if j < OVERLAP { + w[j] + } else if j >= n { + w[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + block[j] * g + }; + // Fold to a DCT-IV input: v = (-c_r - d, a - b_r) over the four quarters of u. + for m in 0..n2 { + self.v[m] = -u(n + n2 - 1 - m) - u(n + n2 + m); + self.v[n2 + m] = u(m) - u(n2 + n2 - 1 - m); + } + // DCT-IV via an n/2-point complex FFT. + self.z.clear(); + for m in 0..n2 { + let wv = Cpx { re: self.v[2 * m], im: self.v[n - 1 - 2 * m] }; + self.z.push(wv.mul(self.pre[m])); + } + self.fft.process(&mut self.z, &mut self.scratch); + for p in 0..n2 { + let y = self.z[p].mul(self.post[p]); + out[2 * p] = y.re * self.scale; + out[n - 1 - 2 * p] = -y.im * self.scale; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Direct MDCT definition X[k] = sum u[t] cos(pi/N (t + 1/2 + N/2)(k + 1/2)). + fn direct(block: &[f32], n: usize, window: &[f32; OVERLAP]) -> Vec { + let pad = (n - OVERLAP) / 2; + let mut u = vec![0f64; 2 * n]; + for (j, &b) in block.iter().enumerate() { + let g = if j < OVERLAP { + window[j] + } else if j >= n { + window[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + u[pad + j] = (b * g) as f64; + } + (0..n) + .map(|k| { + (0..2 * n).map(|t| u[t] * (std::f64::consts::PI / n as f64 * (t as f64 + 0.5 + n as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum::() * 2.0 + / n as f64 + }) + .collect() + } + + #[test] + fn matches_direct_formula() { + for &n in &[240usize, 960] { + let block: Vec = (0..n + OVERLAP).map(|i| ((i * 37 % 101) as f32 - 50.0) / 7.0).collect(); + let mut m = Mdct::new(n); + let mut out = vec![0f32; n]; + m.forward(&block, &mut out); + let d = direct(&block, n, &m.window); + for k in 0..n { + assert!((out[k] as f64 - d[k]).abs() < 1e-2, "n={n} k={k}: {} vs {}", out[k], d[k]); + } + } + } + + // ---- Replica of the RFC 6716 decoder inverse MDCT (as in FilmCraft's decoder) ---- + + fn imdct_backward(n_full: usize, input: &[f32], out: &mut [f32], window: &[f32]) { + let n2 = n_full >> 1; + let n4 = n_full >> 2; + let t: Vec = (0..n2).map(|i| (2.0 * std::f64::consts::PI * (i as f64 + 0.125) / n_full as f64).cos() as f32).collect(); + let fft = Fft::new(n4); + let mut z = Vec::new(); + for i in 0..n4 { + let x1 = input[2 * i]; + let x2 = input[n2 - 1 - 2 * i]; + let yr = x2 * t[i] + x1 * t[n4 + i]; + let yi = x1 * t[i] - x2 * t[n4 + i]; + z.push(Cpx { re: yi, im: yr }); + } + fft.process(&mut z, &mut Vec::new()); + let overlap = OVERLAP; + let base = overlap >> 1; + let y = &mut out[base..base + n2]; + for i in 0..n4.div_ceil(2) { + let front = z[i]; + let back = z[n4 - 1 - i]; + let (re, im) = (front.im, front.re); + let (t0, t1) = (t[i], t[n4 + i]); + let yr0 = re * t0 + im * t1; + let yi0 = re * t1 - im * t0; + let (re, im) = (back.im, back.re); + let (t0, t1) = (t[n4 - i - 1], t[n2 - i - 1]); + let yr1 = re * t0 + im * t1; + let yi1 = re * t1 - im * t0; + y[2 * i] = yr0; + y[n2 - 1 - 2 * i] = yi0; + y[n2 - 2 - 2 * i] = yr1; + y[2 * i + 1] = yi1; + } + for i in 0..overlap / 2 { + let x1 = out[overlap - 1 - i]; + let x2 = out[i]; + let w1 = window[i]; + let w2 = window[overlap - 1 - i]; + out[i] = x2 * w2 - x1 * w1; + out[overlap - 1 - i] = x2 * w1 + x1 * w2; + } + } + + /// MDCT analysis followed by the decoder's IMDCT + TDAC overlap-add reproduces the input + /// delayed by exactly `OVERLAP` samples. + #[test] + fn perfect_reconstruction_through_decoder_imdct() { + let n = 960; + let frames = 5; + let sig: Vec = (0..n * frames).map(|i| (i as f32 * 0.013).sin() * 1000.0 + ((i * 7919) % 211) as f32 - 105.0).collect(); + let mut m = Mdct::new(n); + let mut hist = vec![0f32; OVERLAP]; + // Decoder memory: [frame output n][folded tail OVERLAP/2] laid out like decode_mem. + let mut mem = vec![0f32; n + OVERLAP]; + let mut decoded = Vec::new(); + for f in 0..frames { + let mut block = hist.clone(); + block.extend_from_slice(&sig[f * n..(f + 1) * n]); + hist.copy_from_slice(&block[n..]); + let mut coefs = vec![0f32; n]; + m.forward(&block, &mut coefs); + // Shift: the previous folded tail (at n..n+OVERLAP/2) moves to 0..OVERLAP/2. + mem.copy_within(n..n + OVERLAP / 2, 0); + imdct_backward(2 * n, &coefs, &mut mem, &m.window); + decoded.extend_from_slice(&mem[..n]); + } + let mut err = 0f64; + let mut sum = 0f64; + for t in OVERLAP..n * frames { + let d = decoded[t] as f64 - sig[t - OVERLAP] as f64; + err += d * d; + sum += (sig[t - OVERLAP] as f64).powi(2); + } + let snr = 10.0 * (sum / err).log10(); + assert!(snr > 80.0, "reconstruction SNR {snr:.1} dB"); + } +} diff --git a/crates/opusenc/src/range.rs b/crates/opusenc/src/range.rs new file mode 100644 index 000000000..daedd58ae --- /dev/null +++ b/crates/opusenc/src/range.rs @@ -0,0 +1,472 @@ +//! Range encoder (RFC 6716 §5.1), the exact inverse of the §4.1 range decoder. +//! +//! Range-coded symbols are written from the front of the buffer and raw bits from the back; the +//! bit accounting ([`RangeEncoder::tell`], [`RangeEncoder::tell_frac`]) matches the decoder's, so +//! every budget decision the decoder takes can be mirrored exactly. + +const SYM_BITS: u32 = 8; +const CODE_BITS: u32 = 32; +const SYM_MAX: u32 = (1 << SYM_BITS) - 1; +const CODE_SHIFT: u32 = CODE_BITS - SYM_BITS - 1; +const CODE_TOP: u32 = 1 << (CODE_BITS - 1); +const CODE_BOT: u32 = CODE_TOP >> SYM_BITS; +/// Bit resolution of [`RangeEncoder::tell_frac`] (1/8 bit). +pub const BITRES: u32 = 3; + +/// Number of bits needed to represent `x` (0 for 0). +#[inline] +pub fn ilog(x: u32) -> i32 { + (32 - x.leading_zeros()) as i32 +} + +#[derive(Clone, Debug)] +pub struct RangeEncoder { + out: Vec, + end: Vec, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + low: u32, + rem: i32, + ext: u32, +} + +impl Default for RangeEncoder { + fn default() -> Self { + Self::new() + } +} + +impl RangeEncoder { + pub fn new() -> Self { + RangeEncoder { + out: Vec::new(), + end: Vec::new(), + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1) as i32, + rng: CODE_TOP, + low: 0, + rem: -1, + ext: 0, + } + } + + /// Current range (equals the decoder's range after the same symbols). + #[cfg(test)] + pub fn rng(&self) -> u32 { + self.rng + } + + fn carry_out(&mut self, c: u32) { + if c != SYM_MAX { + let carry = c >> SYM_BITS; + if self.rem >= 0 { + self.out.push((self.rem as u32 + carry) as u8); + } + if self.ext > 0 { + let sym = (SYM_MAX + carry) & SYM_MAX; + for _ in 0..self.ext { + self.out.push(sym as u8); + } + self.ext = 0; + } + self.rem = (c & SYM_MAX) as i32; + } else { + self.ext += 1; + } + } + + #[inline] + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.carry_out(self.low >> CODE_SHIFT); + self.low = (self.low << SYM_BITS) & (CODE_TOP - 1); + self.rng <<= SYM_BITS; + self.nbits_total += SYM_BITS as i32; + } + } + + /// Encodes the symbol with cumulative range `[fl, fh)` of total `ft`. + pub fn encode(&mut self, fl: u32, fh: u32, ft: u32) { + let r = self.rng / ft; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Like [`Self::encode`] with `ft = 1 << bits`. + pub fn encode_bin(&mut self, fl: u32, fh: u32, bits: u32) { + let r = self.rng >> bits; + let ft = 1u32 << bits; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Encodes a binary symbol whose probability of being 1 is `1/2^logp`. + pub fn bit_logp(&mut self, val: bool, logp: u32) { + let r = self.rng; + let s = r >> logp; + let r2 = r - s; + if val { + self.low += r2; + self.rng = s; + } else { + self.rng = r2; + } + self.normalize(); + } + + /// Encodes symbol `s` with an "inverse CDF" table (`ft = 1 << ftb`). + pub fn icdf(&mut self, s: usize, icdf: &[u8], ftb: u32) { + let r = self.rng >> ftb; + if s > 0 { + self.low += self.rng - r * icdf[s - 1] as u32; + self.rng = r * (icdf[s - 1] as u32 - icdf[s] as u32); + } else { + self.rng -= r * icdf[s] as u32; + } + self.normalize(); + } + + /// Encodes a uniformly distributed integer `fl` in `[0, ft)` (`ft >= 2`). + pub fn uint(&mut self, fl: u32, ft: u32) { + debug_assert!(ft > 1 && fl < ft); + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let v = fl >> ftb; + self.encode(v, v + 1, top); + self.bits(fl & ((1 << ftb) - 1), ftb); + } else { + self.encode(fl, fl + 1, ft); + } + } + + /// Writes `bits` raw bits (0..=25) to the end of the buffer. + pub fn bits(&mut self, val: u32, bits: u32) { + if bits == 0 { + return; + } + let mut window = self.end_window; + let mut used = self.nend_bits; + if used as u32 + bits > CODE_BITS { + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + } + window |= val << used; + used += bits as i32; + self.end_window = window; + self.nend_bits = used; + self.nbits_total += bits as i32; + } + + /// Number of bits written so far (rounded up), identical to the decoder's `tell`. + #[inline] + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + + /// Number of 1/8 bits written so far (rounded up), identical to the decoder's `tell_frac`. + pub fn tell_frac(&self) -> i32 { + let nbits = (self.nbits_total as u32) << BITRES; + let mut l = ilog(self.rng) as u32; + let mut r = self.rng >> (l - 16); + for _ in 0..BITRES { + r = (r * r) >> 15; + let b = r >> 16; + l = (l << 1) | b; + r >>= b; + } + nbits.wrapping_sub(l) as i32 + } + + /// Finalises the stream into a buffer of exactly `size` bytes (range data at the front, raw + /// bits at the back, zero padding in between; a partial raw-bit byte is merged into the byte + /// just before the full raw-bit bytes). + pub fn done(mut self, size: usize) -> Vec { + let mut l = CODE_BITS as i32 - ilog(self.rng); + let mut msk = (CODE_TOP - 1) >> l; + let mut end = (self.low.wrapping_add(msk)) & !msk; + if (end | msk) >= self.low.wrapping_add(self.rng) { + l += 1; + msk >>= 1; + end = (self.low.wrapping_add(msk)) & !msk; + } + while l > 0 { + self.carry_out(end >> CODE_SHIFT); + end = (end << SYM_BITS) & (CODE_TOP - 1); + l -= SYM_BITS as i32; + } + if self.rem >= 0 || self.ext > 0 { + self.carry_out(0); + } + let mut window = self.end_window; + let mut used = self.nend_bits; + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + let mut buf = vec![0u8; size]; + let full = self.end.len().min(size); + let room = size - full; + let n = self.out.len().min(room); + buf[..n].copy_from_slice(&self.out[..n]); + for (k, &b) in self.end.iter().take(full).enumerate() { + buf[size - 1 - k] = b; + } + if used > 0 && room > 0 { + buf[room - 1] |= window as u8; + } + buf + } +} + +#[cfg(test)] +pub(crate) mod dec { + //! Minimal range decoder (RFC 6716 §4.1) used to verify the encoder. + use super::*; + const CODE_EXTRA: u32 = (CODE_BITS - 2) % SYM_BITS + 1; + + pub struct RangeDecoder<'a> { + buf: &'a [u8], + offs: usize, + end_offs: usize, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + val: u32, + rem: u32, + } + + impl<'a> RangeDecoder<'a> { + pub fn new(buf: &'a [u8]) -> Self { + let mut d = RangeDecoder { + buf, + offs: 0, + end_offs: 0, + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1 - ((CODE_BITS - CODE_EXTRA) / SYM_BITS) * SYM_BITS) as i32, + rng: 1 << CODE_EXTRA, + val: 0, + rem: 0, + }; + d.rem = d.read_byte(); + d.val = d.rng - 1 - (d.rem >> (SYM_BITS - CODE_EXTRA)); + d.normalize(); + d + } + fn read_byte(&mut self) -> u32 { + if self.offs < self.buf.len() { + self.offs += 1; + self.buf[self.offs - 1] as u32 + } else { + 0 + } + } + fn read_byte_from_end(&mut self) -> u32 { + if self.end_offs < self.buf.len() { + self.end_offs += 1; + self.buf[self.buf.len() - self.end_offs] as u32 + } else { + 0 + } + } + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.nbits_total += SYM_BITS as i32; + self.rng <<= SYM_BITS; + let sym = self.rem; + self.rem = self.read_byte(); + let sym = ((sym << SYM_BITS) | self.rem) >> (SYM_BITS - CODE_EXTRA); + self.val = ((self.val << SYM_BITS).wrapping_add(SYM_MAX & !sym)) & (CODE_TOP - 1); + } + } + pub fn rng(&self) -> u32 { + self.rng + } + pub fn decode(&mut self, ft: u32) -> u32 { + let ext = self.rng / ft; + let s = self.val / ext; + ft - (s + 1).min(ft) + } + pub fn update(&mut self, fl: u32, fh: u32, ft: u32) { + let ext = self.rng / ft; + let s = ext * (ft - fh); + self.val = self.val.wrapping_sub(s); + self.rng = if fl > 0 { ext * (fh - fl) } else { self.rng - s }; + self.normalize(); + } + pub fn bit_logp(&mut self, logp: u32) -> bool { + let r = self.rng; + let d = self.val; + let s = r >> logp; + let ret = d < s; + if !ret { + self.val = d - s; + } + self.rng = if ret { s } else { r - s }; + self.normalize(); + ret + } + pub fn icdf(&mut self, icdf: &[u8], ftb: u32) -> usize { + let mut s = self.rng; + let d = self.val; + let r = s >> ftb; + let mut ret = 0usize; + let mut t; + loop { + t = s; + s = r * icdf[ret] as u32; + if d >= s { + break; + } + ret += 1; + } + self.val = d - s; + self.rng = t - s; + self.normalize(); + ret + } + pub fn uint(&mut self, ft: u32) -> u32 { + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let s = self.decode(top); + self.update(s, s + 1, top); + (s << ftb) | self.bits(ftb) + } else { + let s = self.decode(ft); + self.update(s, s + 1, ft); + s + } + } + pub fn bits(&mut self, bits: u32) -> u32 { + let mut window = self.end_window; + let mut available = self.nend_bits; + if (available as u32) < bits { + loop { + window |= self.read_byte_from_end() << available; + available += SYM_BITS as i32; + if available > (CODE_BITS - SYM_BITS) as i32 { + break; + } + } + } + let ret = window & ((1u32 << bits) - 1); + self.end_window = window >> bits; + self.nend_bits = available - bits as i32; + self.nbits_total += bits as i32; + ret + } + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + } +} + +#[cfg(test)] +mod tests { + use super::dec::RangeDecoder; + use super::*; + + struct Lcg(u32); + impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(1664525).wrapping_add(1013904223); + self.0 >> 8 + } + } + + #[test] + fn starts_at_one_bit() { + let e = RangeEncoder::new(); + assert_eq!(e.tell(), 1); + assert_eq!(e.tell_frac(), 8); + } + + #[test] + fn roundtrip_mixed_symbols_and_tell_matches() { + let mut rng = Lcg(7); + let icdf: [u8; 4] = [200, 120, 30, 0]; + for trial in 0..300 { + let n = 1 + (rng.next() % 300) as usize; + let mut ops = Vec::new(); + for _ in 0..n { + match rng.next() % 5 { + 0 => { + let ft = 2 + rng.next() % 1000; + ops.push((0u8, rng.next() % ft, ft)); + } + 1 => { + let logp = 1 + rng.next() % 15; + ops.push((1, rng.next().is_multiple_of(5) as u32, logp)); + } + 2 => { + let bits = 1 + rng.next() % 24; + ops.push((2, rng.next() & ((1 << bits) - 1), bits)); + } + 3 => { + let ft = 2 + rng.next() % 100_000_000; + ops.push((3, rng.next() % ft, ft)); + } + _ => ops.push((4, rng.next() % 4, 0)), + } + } + let mut e = RangeEncoder::new(); + let mut tells = Vec::new(); + for &(k, v, p) in &ops { + match k { + 0 => e.encode(v, v + 1, p), + 1 => e.bit_logp(v != 0, p), + 2 => e.bits(v, p), + 3 => e.uint(v, p), + _ => e.icdf(v as usize, &icdf, 8), + } + tells.push(e.tell()); + } + // Exactly the number of bytes `tell` says were used must suffice. + let size = (e.tell() as usize).div_ceil(8); + let final_rng = e.rng(); + let buf = e.done(size); + assert_eq!(buf.len(), size); + let mut d = RangeDecoder::new(&buf); + for (i, &(k, v, p)) in ops.iter().enumerate() { + let got = match k { + 0 => { + let s = d.decode(p); + d.update(s, s + 1, p); + s + } + 1 => d.bit_logp(p) as u32, + 2 => d.bits(p), + 3 => d.uint(p), + _ => d.icdf(&icdf, 8) as u32, + }; + assert_eq!(got, v, "trial {trial} op {i} kind {k}"); + assert_eq!(d.tell(), tells[i], "tell mismatch trial {trial} op {i}"); + } + assert_eq!(d.rng(), final_rng); + } + } +} diff --git a/crates/opusenc/src/rate.rs b/crates/opusenc/src/rate.rs new file mode 100644 index 000000000..d8665bf13 --- /dev/null +++ b/crates/opusenc/src/rate.rs @@ -0,0 +1,561 @@ +//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1). +//! +//! The allocation must be bit-identical to the decoder's, so this mirrors FilmCraft's +//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only +//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::tables::*; + +const MAX_PSEUDO: i32 = 40; +const LOG_MAX_PSEUDO: i32 = 6; +pub const MAX_FINE_BITS: i32 = 8; +const FINE_OFFSET: i32 = 21; +pub const QTHETA_OFFSET: i32 = 4; +pub const QTHETA_OFFSET_TWOPHASE: i32 = 16; +const ALLOC_STEPS: i32 = 6; + +/// Number of pulses represented by pseudo-pulse index `i`. +pub fn get_pulses(i: i32) -> i32 { + if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) } +} + +/// Conservative (rounded up) log2 of `val` with `frac` fractional bits. +pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 { + let mut l = ilog(val); + if val & val.wrapping_sub(1) != 0 { + if l > 16 { + val = ((val - 1) >> (l - 16)) + 1; + } else { + val <<= 16 - l; + } + let mut l = (l - 1) << frac; + loop { + let b = (val >> 16) as i32; + l += b << frac; + val = (val + b as u32) >> b; + val = (val * val + 0x7FFF) >> 15; + if frac == 0 { + break; + } + frac -= 1; + } + l + (val > 0x8000) as i32 + } else { + l -= 1; + l << frac + } +} + +/// PVQ codebook size V(N, K) (saturating far above 2^32). +pub fn pvq_v(n: usize, k: usize) -> u128 { + if k == 0 { + return 1; + } + if n == 0 { + return 0; + } + let mut row = vec![0u128; k + 1]; + row[0] = 1; + for _ in 0..n { + let mut prev_diag = row[0]; + for kk in 1..=k { + let up = row[kk]; + let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag); + prev_diag = up; + row[kk] = v.min(u128::MAX >> 8); + } + row[0] = 1; + } + row[k] +} + +fn fits_in32(n: usize, k: i32) -> bool { + pvq_v(n, k as usize) <= u32::MAX as u128 +} + +/// Precomputed per-mode data. +pub struct Mode { + /// log2 of band width in 1/8 bits. + pub log_n: [i32; NB_EBANDS], + cache_index: Vec, + cache_bits: Vec, + caps: Vec, +} + +impl Mode { + pub fn new() -> Mode { + let mut log_n = [0i32; NB_EBANDS]; + for (i, l) in log_n.iter_mut().enumerate() { + *l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES); + } + let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32); + let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() }; + m.caps = compute_caps(&m); + m + } + + #[inline] + pub fn cache(&self, lm: i32, band: usize) -> &[u8] { + let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band]; + &self.cache_bits[idx.max(0) as usize..] + } + + pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 { + let cache = self.cache(lm, band); + let mut lo = 0i32; + let mut hi = cache[0] as i32; + let bits = bits - 1; + for _ in 0..LOG_MAX_PSEUDO { + let mid = (lo + hi + 1) >> 1; + if cache[mid as usize] as i32 >= bits { + hi = mid; + } else { + lo = mid; + } + } + let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 }; + if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi } + } + + pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 { + if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 } + } + + /// Per-band allocation caps for this frame (`init_caps`). + pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] { + let mut cap = [0i32; NB_EBANDS]; + for (i, cp) in cap.iter_mut().enumerate() { + let n = (EBANDS[i + 1] - EBANDS[i]) << lm; + *cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2; + } + cap + } +} + +fn compute_pulse_cache(lm_max: i32) -> (Vec, Vec) { + let nb = NB_EBANDS; + let rows = (lm_max + 2) as usize; + let mut index = vec![-1i16; nb * rows]; + let mut entries: Vec<(usize, i32, usize)> = Vec::new(); + let mut curr = 0usize; + for i in 0..rows { + for j in 0..nb { + let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize; + index[i * nb + j] = -1; + 'search: for k in 0..=i { + for nn in 0..nb { + if k == i && nn >= j { + break; + } + if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize { + index[i * nb + j] = index[k * nb + nn]; + break 'search; + } + } + } + if index[i * nb + j] == -1 && n != 0 { + let mut k = 0; + while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) { + k += 1; + } + index[i * nb + j] = curr as i16; + entries.push((n, k, curr)); + curr += (k + 1) as usize; + } + } + } + let mut bits = vec![0u8; curr]; + for &(n, k, off) in &entries { + let maxk = get_pulses(k) as usize; + let tmp = required_bits(n, maxk, BITRES); + for j in 1..=k { + bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8; + } + bits[off] = k as u8; + } + (index, bits) +} + +fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec { + let mut out = vec![0i32; maxk + 1]; + if n == 1 { + for o in out.iter_mut().skip(1) { + *o = 1 << frac; + } + } else { + for (k, o) in out.iter_mut().enumerate().skip(1) { + *o = log2_frac(pvq_v(n, k) as u32, frac); + } + } + out +} + +fn compute_caps(m: &Mode) -> Vec { + let nb = NB_EBANDS; + let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb]; + for i in 0..=MAX_LM as i32 { + for c in 1..=2i32 { + for j in 0..nb { + let mut n0 = EBANDS[j + 1] - EBANDS[j]; + let max_bits; + if n0 << i == 1 { + max_bits = c * (1 + MAX_FINE_BITS) << BITRES; + } else { + let mut lm0 = 0; + if n0 > 2 { + n0 >>= 1; + lm0 -= 1; + } else if n0 <= 1 { + lm0 = i.min(1); + n0 <<= lm0; + } + let pcache = m.cache(lm0, j); + let mut mb = pcache[pcache[0] as usize] as i32 + 1; + let mut n = n0; + for k in 0..(i - lm0) { + mb <<= 1; + let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET; + let num = 459 * ((2 * n - 1) * offset + mb); + let den = ((2 * n - 1) << 9) - 459; + let qb = ((num + (den >> 1)) / den).min(57); + mb += qb; + n <<= 1; + } + if c == 2 { + mb <<= 1; + let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET }; + let ndof = 2 * n - 1 - (n == 2) as i32; + let p = if n == 2 { 512 } else { 487 }; + let num = p * (mb + ndof * offset); + let den = (ndof << 9) - p; + let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 }); + mb += qb; + } + let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 }; + let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET; + if n == 2 { + offset += (1 << BITRES) >> 2; + } + let num = mb + ndof * offset; + let den = (ndof - 1) << BITRES; + let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS); + mb += c * qb << BITRES; + max_bits = mb; + } + let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i); + let v = (4 * max_bits / w) - 64; + caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8; + } + } + } + caps +} + +/// Result of the allocation process. +pub struct Allocation { + pub coded_bands: usize, + pub intensity: usize, + pub dual_stereo: bool, + pub balance: i32, + pub pulses: [i32; NB_EBANDS], + pub fine_quant: [i32; NB_EBANDS], + pub fine_priority: [i32; NB_EBANDS], +} + +/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo +/// with no intensity bands, and no band is skipped voluntarily. +#[allow(clippy::too_many_arguments)] +pub fn compute_allocation( + m: &Mode, + start: usize, + end: usize, + offsets: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + alloc_trim: i32, + total: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let lmi = lm as i32; + let mut total = total.max(0); + let mut skip_start = start; + let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= skip_rsv; + let mut intensity_rsv = 0; + let mut dual_stereo_rsv = 0; + if c == 2 { + intensity_rsv = LOG2_FRAC_TABLE[end - start]; + if intensity_rsv > total { + intensity_rsv = 0; + } else { + total -= intensity_rsv; + dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= dual_stereo_rsv; + } + } + let mut bits1 = [0i32; NB_EBANDS]; + let mut bits2 = [0i32; NB_EBANDS]; + let mut thresh = [0i32; NB_EBANDS]; + let mut trim_offset = [0i32; NB_EBANDS]; + for j in start..end { + let w = EBANDS[j + 1] - EBANDS[j]; + thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4); + trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6; + if w << lmi == 1 { + trim_offset[j] -= ci << BITRES; + } + } + let nb_alloc = ALLOC.len() as i32; + let mut lo = 1; + let mut hi = nb_alloc - 1; + loop { + let mut done = false; + let mut psum = 0; + let mid = (lo + hi) >> 1; + for j in (start..end).rev() { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2; + if bitsj > 0 { + bitsj = (bitsj + trim_offset[j]).max(0); + } + bitsj += offsets[j]; + if bitsj >= thresh[j] || done { + done = true; + psum += bitsj.min(cap[j]); + } else if bitsj >= ci << BITRES { + psum += ci << BITRES; + } + } + if psum > total { + hi = mid - 1; + } else { + lo = mid + 1; + } + if lo > hi { + break; + } + } + hi = lo; + lo -= 1; + for j in start..end { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2; + let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 }; + if bits1j > 0 { + bits1j = (bits1j + trim_offset[j]).max(0); + } + if bits2j > 0 { + bits2j = (bits2j + trim_offset[j]).max(0); + } + if lo > 0 { + bits1j += offsets[j]; + } + bits2j += offsets[j]; + if offsets[j] > 0 { + skip_start = j; + } + bits2j = (bits2j - bits1j).max(0); + bits1[j] = bits1j; + bits2[j] = bits2j; + } + interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc) +} + +#[allow(clippy::too_many_arguments)] +fn interp_bits2pulses( + m: &Mode, + start: usize, + end: usize, + skip_start: usize, + bits1: &[i32; NB_EBANDS], + bits2: &[i32; NB_EBANDS], + thresh: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + mut total: i32, + skip_rsv: i32, + mut intensity_rsv: i32, + mut dual_stereo_rsv: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let alloc_floor = ci << BITRES; + let stereo = (c > 1) as i32; + let log_m = (lm as i32) << BITRES; + let mut lo = 0; + let mut hi = 1 << ALLOC_STEPS; + for _ in 0..ALLOC_STEPS { + let mid = (lo + hi) >> 1; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS); + if tmp >= thresh[j] || done { + done = true; + psum += tmp.min(cap[j]); + } else if tmp >= alloc_floor { + psum += alloc_floor; + } + } + if psum > total { + hi = mid; + } else { + lo = mid; + } + } + let mut bits = [0i32; NB_EBANDS]; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS); + if tmp < thresh[j] && !done { + tmp = if tmp >= alloc_floor { alloc_floor } else { 0 }; + } else { + done = true; + } + tmp = tmp.min(cap[j]); + bits[j] = tmp; + psum += tmp; + } + let s = start; + let mut coded_bands = end; + loop { + let j = coded_bands - 1; + if j <= skip_start { + total += skip_rsv; + break; + } + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + let rem = (left - (EBANDS[j] - EBANDS[s])).max(0); + let band_width = EBANDS[coded_bands] - EBANDS[j]; + let band_bits = bits[j] + percoeff * band_width + rem; + if band_bits >= thresh[j].max(alloc_floor + (1 << BITRES)) { + // Keep this band (and every band below it). + enc.bit_logp(true, 1); + break; + } + psum -= bits[j] + intensity_rsv; + if intensity_rsv > 0 { + intensity_rsv = LOG2_FRAC_TABLE[j - s]; + } + psum += intensity_rsv; + if band_bits >= alloc_floor { + psum += alloc_floor; + bits[j] = alloc_floor; + } else { + bits[j] = 0; + } + coded_bands -= 1; + } + let intensity = if intensity_rsv > 0 { + // No intensity stereo: every coded band is coded in full. + enc.uint((coded_bands - s) as u32, (coded_bands + 1 - s) as u32); + coded_bands + } else { + 0 + }; + if intensity <= s { + total += dual_stereo_rsv; + dual_stereo_rsv = 0; + } + let dual_stereo = if dual_stereo_rsv > 0 { + enc.bit_logp(true, 1); + true + } else { + false + }; + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + for j in s..coded_bands { + bits[j] += percoeff * (EBANDS[j + 1] - EBANDS[j]); + } + for j in s..coded_bands { + let tmp = left.min(EBANDS[j + 1] - EBANDS[j]); + bits[j] += tmp; + left -= tmp; + } + let mut ebits = [0i32; NB_EBANDS]; + let mut fine_priority = [0i32; NB_EBANDS]; + let mut balance = 0; + for j in s..coded_bands { + let n0 = EBANDS[j + 1] - EBANDS[j]; + let n = n0 << lm; + let bit = bits[j] + balance; + let mut excess; + if n > 1 { + excess = (bit - cap[j]).max(0); + bits[j] = bit - excess; + let den = ci * n + if c == 2 && n > 2 && !dual_stereo && j < intensity { 1 } else { 0 }; + let nclogn = den * (m.log_n[j] + log_m); + let mut offset = (nclogn >> 1) - den * FINE_OFFSET; + if n == 2 { + offset += (den << BITRES) >> 2; + } + if bits[j] + offset < den * 2 << BITRES { + offset += nclogn >> 2; + } else if bits[j] + offset < den * 3 << BITRES { + offset += nclogn >> 3; + } + let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0); + eb = (eb / den) >> BITRES; + if ci * eb > (bits[j] >> BITRES) { + eb = bits[j] >> stereo >> BITRES; + } + eb = eb.min(MAX_FINE_BITS); + ebits[j] = eb; + fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32; + bits[j] -= ci * eb << BITRES; + } else { + excess = (bit - (ci << BITRES)).max(0); + bits[j] = bit - excess; + ebits[j] = 0; + fine_priority[j] = 1; + } + if excess > 0 { + let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]); + ebits[j] += extra_fine; + let extra_bits = extra_fine * ci << BITRES; + fine_priority[j] = (extra_bits >= excess - balance) as i32; + excess -= extra_bits; + } + balance = excess; + } + for j in coded_bands..end { + ebits[j] = bits[j] >> stereo >> BITRES; + bits[j] = 0; + fine_priority[j] = (ebits[j] < 1) as i32; + } + Allocation { coded_bands, intensity, dual_stereo, balance, pulses: bits, fine_quant: ebits, fine_priority } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caps_match_decoder_reference() { + let m = Mode::new(); + let first_rows: [u8; 42] = [ + 224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, 178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, + 240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, + ]; + assert_eq!(&m.caps[..42], &first_rows[..]); + } + + #[test] + fn pvq_counts() { + assert_eq!(pvq_v(5, 1), 10); + assert_eq!(pvq_v(1, 7), 2); + assert_eq!(pvq_v(2, 9), 36); + assert_eq!(pvq_v(3, 2), 18); + } +} diff --git a/crates/opusenc/src/tables.rs b/crates/opusenc/src/tables.rs new file mode 100644 index 000000000..00d623af0 --- /dev/null +++ b/crates/opusenc/src/tables.rs @@ -0,0 +1,122 @@ +//! Constant tables of the 48 kHz CELT mode (RFC 6716 §4.3, Tables 55–58 and §4.3.2). +//! +//! Transcribed from RFC 6716; copied from FilmCraft's spec-derived `filmcraft-opus` decoder +//! (first-party, MIT OR Apache-2.0) so both sides share the exact same values. + +/// Number of energy bands. +pub const NB_EBANDS: usize = 21; +/// Band edges in units of 2.5 ms MDCT bins (Table 55). +pub const EBANDS: [i32; NB_EBANDS + 1] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100]; +/// Overlap of the low-overlap window (samples at 48 kHz). +pub const OVERLAP: usize = 120; +/// Largest LM (20 ms frames). +pub const MAX_LM: usize = 3; +/// Pre-emphasis coefficient. +pub const PREEMPH: f32 = 0.850_006_1; + +/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin. +pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([ + [0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200], + [0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200], + [0, 75, 90, 103, 112, 120, 130, 138, 148, 158, 200], + [0, 69, 84, 93, 104, 114, 124, 132, 142, 152, 200], + [0, 63, 78, 86, 95, 103, 113, 123, 133, 143, 200], + [0, 56, 71, 80, 89, 97, 107, 117, 127, 137, 200], + [0, 49, 65, 75, 83, 91, 101, 111, 121, 131, 200], + [0, 40, 58, 70, 78, 85, 95, 105, 115, 125, 200], + [0, 34, 51, 65, 72, 78, 88, 98, 108, 118, 198], + [0, 29, 45, 59, 66, 72, 82, 92, 102, 112, 193], + [0, 20, 39, 53, 60, 66, 76, 86, 96, 106, 188], + [0, 18, 32, 47, 54, 60, 70, 80, 90, 100, 183], + [0, 10, 26, 40, 47, 54, 64, 74, 84, 94, 178], + [0, 0, 20, 31, 39, 47, 57, 67, 77, 87, 173], + [0, 0, 12, 23, 32, 41, 51, 61, 71, 81, 168], + [0, 0, 0, 15, 25, 35, 45, 55, 65, 75, 163], + [0, 0, 0, 4, 17, 29, 39, 49, 59, 69, 158], + [0, 0, 0, 0, 12, 23, 33, 43, 53, 63, 153], + [0, 0, 0, 0, 1, 16, 26, 36, 46, 56, 148], + [0, 0, 0, 0, 0, 10, 15, 20, 30, 45, 129], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104], +]); + +const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] { + let mut out = [[0u8; NB_EBANDS]; 11]; + let mut b = 0; + while b < NB_EBANDS { + let mut q = 0; + while q < 11 { + out[q][b] = t[b][q]; + q += 1; + } + b += 1; + } + out +} + +/// Mean band energy (log2 domain) removed before coarse quantisation. +pub const E_MEANS: [f32; 25] = [ + 6.4375, 6.25, 5.75, 5.3125, 5.0625, 4.8125, 4.5, 4.375, 4.875, 4.6875, 4.5625, 4.4375, 4.875, 4.625, 4.3125, 4.5, 4.375, 4.625, 4.75, 4.4375, 3.75, 3.75, + 3.75, 3.75, 3.75, +]; + +/// Inter-frame prediction coefficient (alpha) per LM, Q15. +pub const PRED_COEF: [f32; 4] = [29440.0 / 32768.0, 26112.0 / 32768.0, 21248.0 / 32768.0, 16384.0 / 32768.0]; +/// Inter-band prediction coefficient (beta) per LM for inter frames, Q15. +pub const BETA_COEF: [f32; 4] = [30147.0 / 32768.0, 22282.0 / 32768.0, 12124.0 / 32768.0, 6554.0 / 32768.0]; +/// Beta for intra frames. +pub const BETA_INTRA: f32 = 4915.0 / 32768.0; + +/// Laplace model parameters for coarse energy: `[LM][intra][2*band + {0: prob, 1: decay}]`. +pub const E_PROB_MODEL: [[[u8; 42]; 2]; 4] = [ + [ + [ + 72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, 114, 40, 132, 26, 132, + 26, 145, 17, 161, 12, 176, 10, 177, 11, + ], + [ + 24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, 91, 67, 100, 59, 108, 50, + 120, 40, 122, 37, 97, 43, 78, 50, + ], + ], + [ + [ + 83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, 146, 19, 162, 12, 165, 10, + 178, 7, 189, 6, 190, 8, 177, 9, + ], + [ + 23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, 104, 60, 117, 52, 123, 44, + 138, 35, 133, 31, 97, 38, 77, 45, + ], + ], + [ + [ + 61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, 158, 18, 170, 13, 177, + 10, 187, 8, 192, 6, 175, 9, 159, 10, + ], + [ + 21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, 112, 56, 129, 51, 132, 40, + 150, 33, 140, 29, 98, 35, 77, 42, + ], + ], + [ + [ + 42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, 154, 26, 166, 21, 173, + 16, 184, 13, 184, 10, 150, 13, 139, 15, + ], + [ + 22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, 117, 55, 135, 49, 137, 39, + 157, 32, 145, 29, 97, 33, 77, 40, + ], + ], +]; + +/// Inverse CDF for coarse energy when only a few bits remain. +pub const SMALL_ENERGY_ICDF: [u8; 3] = [2, 1, 0]; +/// Conservative log2 in 1/8 bits of the number of coded bands (intensity reservation). +pub const LOG2_FRAC_TABLE: [i32; 24] = [0, 8, 13, 16, 19, 21, 23, 24, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 37]; +/// TF resolution changes: `[LM][4*transient + 2*tf_select + tf_flag]`. +pub const TF_SELECT_TABLE: [[i8; 8]; 4] = [[0, -1, 0, -1, 0, -1, 0, -1], [0, -1, 0, -2, 1, 0, 1, -1], [0, -2, 0, -3, 2, 0, 1, -1], [0, -2, 0, -3, 3, 0, 1, -1]]; +/// Spreading decision `{7, 2, 21, 2}/32`. +pub const SPREAD_ICDF: [u8; 4] = [25, 23, 2, 0]; +/// Allocation trim (Table 58). +pub const TRIM_ICDF: [u8; 11] = [126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0]; diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb5..c3d869c01 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,6 +16,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3d..0cef3307d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -24,6 +24,7 @@ const LAYERS: &[(&str, u8)] = &[ ("geom", 0), ("color", 0), ("testkit", 0), + ("opusenc", 0), ("raster", 1), ("keyframe", 1), ("path", 1), From bab4f88c22e525ac69474a8293b155fe70fccab4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:44:51 -0400 Subject: [PATCH 02/14] M12.6: camera tracker solve Structure from motion for the 3D Camera Tracker in effectcraft-track (camtrack): long KLT feature tracks with re-detection, parallax keyframes, two-view initialisation (normalised 8-point essential matrix in RANSAC or planar homography decomposition), incremental resection + triangulation, sparse Levenberg-Marquardt bundle adjustment with the Schur complement and Huber loss, focal length search (Fixed / Variable Zoom / Specify Angle of View), tripod pan solve, Auto Detect, ground plane / target / world mapping. Synthetic-scene tests (path, focal, error, tripod, flat, rendered frames). Co-Authored-By: Claude Opus 5.5 --- crates/track/Cargo.toml | 2 +- crates/track/src/camtrack/bundle.rs | 416 ++++++++++++ crates/track/src/camtrack/features.rs | 115 ++++ crates/track/src/camtrack/geometry.rs | 432 +++++++++++++ crates/track/src/camtrack/linalg.rs | 476 ++++++++++++++ crates/track/src/camtrack/mod.rs | 376 +++++++++++ crates/track/src/camtrack/solver.rs | 894 ++++++++++++++++++++++++++ crates/track/src/camtrack/tests.rs | 360 +++++++++++ crates/track/src/lib.rs | 1 + 9 files changed, 3071 insertions(+), 1 deletion(-) create mode 100644 crates/track/src/camtrack/bundle.rs create mode 100644 crates/track/src/camtrack/features.rs create mode 100644 crates/track/src/camtrack/geometry.rs create mode 100644 crates/track/src/camtrack/linalg.rs create mode 100644 crates/track/src/camtrack/mod.rs create mode 100644 crates/track/src/camtrack/solver.rs create mode 100644 crates/track/src/camtrack/tests.rs diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92b..9bd46e26c 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 000000000..84e8a5ad6 --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,416 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + match ps.w.iter_mut().find(|e| e.0 == *ia) { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points. + let mut vinv: Vec> = vec![None; npts]; + if !o.fix_points { + for (j, ps) in systems.iter().enumerate() { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + let Some(vi) = inv3(&v) else { continue }; + vinv[j] = Some(vi); + let vb = mv(&vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w { + s[ia * n + ib] -= dot(wv[a], *wb); + } + } + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-10 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 000000000..311ec711d --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 000000000..6956ac2b5 --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 000000000..a7067f262 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,476 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j are independent. + use rayon::prelude::*; + let work = |row: &mut [f64]| { + let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); + row[j] = s / d; + }; + if n - j > 192 { + tail.par_chunks_mut(n).for_each(work); + } else { + tail.chunks_mut(n).for_each(work); + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 000000000..056d7ad03 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,376 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one. +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 000000000..da3a92908 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,894 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels). + noise: f64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise + } + fn huber(&self) -> f64 { + 1.0 * self.noise + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared +/// features drop below 24. The first and last frames with features are keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + if try_tripod && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + if cancelled() { + return err("cancelled"); + } + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 000000000..258161f98 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,360 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5f..bd4e21dce 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; From 94e1c86d89a866771f80ad1610afda9aa12e85d8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:10:48 -0400 Subject: [PATCH 03/14] IO-2/SHP-5/PRV-2: PSD import, SVG import + vector/text conversions, persistent disk cache - effectcraft-psd (L1): PSD/PSB reader from Adobe's published file format spec (layers, groups, masks, blend modes, text engine data, lfx2 effects, fill/adjustment layers, vector masks; 1/8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer for fixtures. - File > Import: importAs footage|composition|compositionLayerSizes, layer (Choose Layer); groups -> precomps (pass-through collapses), text -> editable text, styles -> Layer Styles, adjustments -> adjustment layers with effects; UI asks the Import Kind for Photoshop files. - effectcraft-svg (L2): SVG parser (shapes, paths, transforms, use, CSS, gradients) and rasteriser; .svg imports as footage, continuously rasterised when the switch is on. - layer.create: shapesFromVector, shapesFromText, masksFromText, editableText (stub removed). - Disk cache (render::disk_cache): LZ4 content-keyed frames and layer buffers, LRU size limit from prefs, atomic writes, restart persistence; edit.purge clears it, cache.diskStats query; viewer frames read/write it and the timeline shows disk-only frames in blue. Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 2 + Cargo.lock | 44 + Cargo.toml | 2 + crates/engine/Cargo.toml | 3 + crates/engine/src/commands/create.rs | 139 +++ crates/engine/src/commands/edit.rs | 32 +- crates/engine/src/commands/file.rs | 98 +- crates/engine/src/commands/file_more.rs | 1 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 12 + crates/engine/src/psd_import.rs | 634 ++++++++++++ crates/engine/src/session_settings.rs | 37 + crates/engine/src/tests_disk_cache.rs | 121 +++ crates/engine/src/tests_stubs.rs | 1 + crates/engine/src/tests_track.rs | 2 + crates/engine/src/tests_vector_import.rs | 237 +++++ crates/engine/src/vector.rs | 250 +++++ crates/gpu/src/tests.rs | 1 + crates/host/Cargo.toml | 4 + crates/host/tests/vector_import.rs | 163 +++ crates/lottie/src/import.rs | 1 + crates/lottie/src/tests.rs | 1 + crates/media/Cargo.toml | 2 + crates/media/src/convert.rs | 17 + crates/media/src/layered.rs | 62 ++ crates/media/src/lib.rs | 6 +- crates/media/src/pool.rs | 16 + crates/media/src/probe.rs | 7 +- crates/project/src/lib.rs | 15 + crates/psd/Cargo.toml | 13 + crates/psd/README.md | 17 + crates/psd/src/descriptor.rs | 430 ++++++++ crates/psd/src/engine_data.rs | 288 ++++++ crates/psd/src/lib.rs | 965 ++++++++++++++++++ crates/psd/src/read.rs | 86 ++ crates/psd/src/tests.rs | 189 ++++ crates/psd/src/write.rs | 491 +++++++++ crates/render/Cargo.toml | 1 + crates/render/src/audio.rs | 1 + crates/render/src/cache.rs | 62 +- crates/render/src/disk_cache.rs | 829 +++++++++++++++ crates/render/src/lib.rs | 29 +- crates/render/src/tests_audio_fx.rs | 1 + crates/render/src/tests_color.rs | 1 + crates/render/src/tests_frame_blend.rs | 1 + crates/svg/Cargo.toml | 18 + crates/svg/README.md | 26 + crates/svg/src/color.rs | 254 +++++ crates/svg/src/lib.rs | 949 +++++++++++++++++ crates/svg/src/pathdata.rs | 293 ++++++ crates/svg/src/raster.rs | 220 ++++ crates/svg/src/tests.rs | 108 ++ crates/svg/tests/fixtures/basic-shapes.svg | 15 + .../fixtures/basic-shapes.svg.attribution | 7 + crates/svg/tests/fixtures/paths-gradients.svg | 26 + .../fixtures/paths-gradients.svg.attribution | 7 + crates/ui-egui/src/frames.rs | 108 +- crates/ui-egui/src/lib.rs | 18 +- crates/ui-egui/src/menus.rs | 6 +- crates/ui-egui/src/panels/forms.rs | 20 + crates/ui-egui/src/panels/timeline.rs | 25 + crates/ui-egui/src/theme.rs | 3 + xtask/src/main.rs | 2 + 64 files changed, 7388 insertions(+), 34 deletions(-) create mode 100644 crates/engine/src/commands/create.rs create mode 100644 crates/engine/src/psd_import.rs create mode 100644 crates/engine/src/tests_disk_cache.rs create mode 100644 crates/engine/src/tests_vector_import.rs create mode 100644 crates/engine/src/vector.rs create mode 100644 crates/host/tests/vector_import.rs create mode 100644 crates/media/src/layered.rs create mode 100644 crates/psd/Cargo.toml create mode 100644 crates/psd/README.md create mode 100644 crates/psd/src/descriptor.rs create mode 100644 crates/psd/src/engine_data.rs create mode 100644 crates/psd/src/lib.rs create mode 100644 crates/psd/src/read.rs create mode 100644 crates/psd/src/tests.rs create mode 100644 crates/psd/src/write.rs create mode 100644 crates/render/src/disk_cache.rs create mode 100644 crates/svg/Cargo.toml create mode 100644 crates/svg/README.md create mode 100644 crates/svg/src/color.rs create mode 100644 crates/svg/src/lib.rs create mode 100644 crates/svg/src/pathdata.rs create mode 100644 crates/svg/src/raster.rs create mode 100644 crates/svg/src/tests.rs create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg.attribution create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg.attribution diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9caf1c278..d0b936b8b 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 | ## Screenshots diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..1d14bab9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1198,12 +1198,15 @@ dependencies = [ "effectcraft-lottie", "effectcraft-path", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-text", "effectcraft-time", "effectcraft-track", "image", + "kurbo", "log", "rayon", "serde", @@ -1287,6 +1290,10 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-psd", + "effectcraft-raster", + "effectcraft-svg", + "effectcraft-time", "serde_json", ] @@ -1324,8 +1331,10 @@ name = "effectcraft-media" version = "0.1.0" dependencies = [ "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-time", "filmcraft-codecs", "filmcraft-color", @@ -1363,6 +1372,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-psd" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-raster" version = "0.1.0" @@ -1387,11 +1403,24 @@ dependencies = [ "effectcraft-time", "kurbo", "log", + "lz4_flex", "rayon", "serde", "web-time", ] +[[package]] +name = "effectcraft-svg" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "effectcraft-path", + "effectcraft-raster", + "kurbo", + "rayon", + "roxmltree", +] + [[package]] name = "effectcraft-text" version = "0.1.0" @@ -2749,6 +2778,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "mach2" version = "0.4.3" @@ -3970,6 +4005,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf45..7aabc4afb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,8 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-psd = { path = "crates/psd" } +effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 132bdcacd..0830c78d9 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,6 +22,9 @@ effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } effectcraft-lottie = { workspace = true } +effectcraft-psd = { workspace = true } +kurbo = { workspace = true } +effectcraft-svg = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/commands/create.rs b/crates/engine/src/commands/create.rs new file mode 100644 index 000000000..72cb2420f --- /dev/null +++ b/crates/engine/src/commands/create.rs @@ -0,0 +1,139 @@ +//! Layer ▸ Create: Convert to Editable Text (Photoshop type layers), Create Shapes from Text, +//! Create Masks from Text and Create Shapes from Vector Layer (SVG footage). + +use effectcraft_color::Label; +use effectcraft_project::{ItemKind, LayerSource, Solid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, has_layers, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const OPS: &str = "editableText|shapesFromText|masksFromText|shapesFromVector"; + +fn create(s: &mut Session, p: &Value) -> Result { + let op = str_p(p, "op").ok_or_else(|| bad("layer.create", format!("missing `op` ({OPS})")))?; + let (cid, lids) = layers_p(s, p)?; + if lids.is_empty() { + return Err(bad("layer.create", "select a layer first")); + } + let time = s.time(); + let mut created = vec![]; + let mut skipped = vec![]; + for lid in lids { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let Some(src) = comp.layer(lid).cloned() else { continue }; + let footage = src.source.item().and_then(|i| match &s.project.item(i)?.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }); + match op { + "shapesFromVector" => { + let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else { + skipped.push(format!("{}: not a vector (SVG) footage layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?; + let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| { + let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc); + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "shapesFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let snapshot = s.project.clone(); + let id = s.edit("Create Shapes from Text", None, |proj, st| { + let l = crate::vector::shapes_from_text(proj, &comp, &snapshot, cid, &src, time) + .ok_or_else(|| bad("layer.create", "the text layer has no glyphs"))?; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "masksFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let outlines = crate::vector::text_outlines_in_comp(&s.project, cid, &comp, &src, time); + if outlines.is_empty() { + skipped.push(format!("{}: no glyphs", src.name)); + continue; + } + let id = s.edit("Create Masks from Text", None, |proj, st| { + let name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = crate::vector::masks_layer(proj, &comp, &name, outlines, sid); + l.in_point = src.in_point; + l.out_point = src.out_point; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "editableText" => { + let Some((f, sl)) = footage.and_then(|f| f.layer.clone().map(|l| (f, l))) else { + skipped.push(format!("{}: not a Photoshop layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let id = s.edit("Convert to Editable Text", None, |proj, st| { + let l = crate::psd_import::editable_text(proj, &comp, &src, &psd, sl.index as usize) + .ok_or_else(|| bad("layer.create", format!("{}: the Photoshop layer has no text", src.name)))?; + let id = l.id; + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == lid).ok_or(EngineError::NoComp)?; + c.layers[at] = l; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + other => return Err(bad("layer.create", format!("`{other}` is not available yet (supported: {OPS})"))), + } + } + if created.is_empty() && !skipped.is_empty() { + return Err(bad("layer.create", skipped.join("; "))); + } + Ok(json!({"layers": created, "skipped": skipped})) +} + +/// Insert `l` directly above layer `below`; optionally switch the source's video off (as After +/// Effects does after converting). +fn insert_above( + proj: &mut effectcraft_project::Project, + cid: effectcraft_project::ItemId, + below: effectcraft_project::LayerId, + l: effectcraft_project::Layer, + hide_source: bool, +) -> Result<()> { + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == below).unwrap_or(0); + if hide_source && let Some(src) = c.layers.get_mut(at) { + src.switches.video = false; + } + c.layers.insert(at, l); + Ok(()) +} + +pub fn specs() -> Vec { + vec![cmd!("layer.create", "Create", [], None, "{op: editableText|shapesFromText|masksFromText|shapesFromVector, layers?}", has_layers, create)] +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd78..9a134b05f 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,9 +525,38 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + let memory = matches!(what.as_str(), "all" | "memoryAndDisk" | "memory" | "image"); + let disk = matches!(what.as_str(), "all" | "memoryAndDisk" | "disk"); + if memory { + s.layer_cache.clear(); + } + let mut disk_entries = 0; + if disk && let Some(dc) = &s.disk_cache { + disk_entries = dc.stats().entries; + dc.clear(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); - Ok(json!({"purged": what})) + Ok(json!({"purged": what, "diskEntries": disk_entries})) +} + +/// `cache.diskStats`: the disk cache's folder, limit and contents. +pub(crate) fn disk_stats(s: &mut Session, _: &Value) -> Result { + let Some(dc) = &s.disk_cache else { return Ok(json!({"enabled": false})) }; + let st = dc.stats(); + Ok(json!({ + "enabled": true, + "folder": dc.folder().to_string_lossy(), + "entries": st.entries, + "frames": dc.keys(effectcraft_render::disk_cache::Kind::Frame).len(), + "layers": dc.keys(effectcraft_render::disk_cache::Kind::Layer).len(), + "bytes": st.bytes, + "maxBytes": st.max_bytes, + "hits": st.hits, + "misses": st.misses, + "writes": st.writes, + "evictions": st.evictions, + })) } /// Path of the footage behind the selected layer or project item. @@ -596,6 +625,7 @@ pub fn specs() -> Vec { cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group), cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge), cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), + crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), ] } diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index a85a391fe..a176c400f 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,10 +84,48 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; - let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; + // Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes). + let import_as = str_p(p, "importAs").unwrap_or("footage"); + let retain = match import_as { + "footage" => None, + "composition" | "comp" => Some(false), + "compositionLayerSizes" | "compositionRetainLayerSizes" | "layerSizes" => Some(true), + other => return Err(bad("file.import", format!("importAs: footage|composition|compositionLayerSizes, not `{other}`"))), + }; + let mut paths = paths; + let mut out_comps = vec![]; let mut ids = vec![]; let mut errors = vec![]; + if let Some(retain) = retain { + let mut rest = vec![]; + for path in paths { + let bytes = match s.services.read_file(&path) { + Ok(b) if effectcraft_psd::is_psd(&b) => b, + _ => { + rest.push(path); + continue; + } + }; + match import_psd_comp(s, &path, bytes, retain) { + Ok((comp, items, warnings)) => { + out_comps.push(comp); + ids.extend(items); + errors.extend(warnings); + } + Err(e) => errors.push(format!("{path}: {e}")), + } + } + paths = rest; + if paths.is_empty() { + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + return Ok(json!({"items": ids, "comps": out_comps, "errors": errors})); + } + } + let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; let mut probed = vec![]; + let psd_layer = p.get("layer").cloned(); let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { match importer.probe(path) { @@ -99,6 +137,18 @@ fn import(s: &mut Session, p: &Value) -> Result { f.frame_rate = r; f.duration = r.tick_of(frames.max(1)); } + // Choose Layer: one layer of a Photoshop document (document-sized). + if let Some(sel) = &psd_layer + && f.codec == "PSD" + { + match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) { + Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }), + None => { + errors.push(format!("{path}: no layer {sel}")); + continue; + } + } + } probed.push((path.clone(), f)) } Err(e) => errors.push(format!("{path}: {e}")), @@ -107,6 +157,10 @@ fn import(s: &mut Session, p: &Value) -> Result { s.edit("Import", None, |proj, st| { for (path, f) in probed { let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); + let name = match &f.layer { + Some(l) => format!("{}/{name}", l.name), + None => name, + }; let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, @@ -118,7 +172,37 @@ fn import(s: &mut Session, p: &Value) -> Result { } Ok(()) })?; - Ok(json!({"items": ids, "errors": errors})) + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + Ok(json!({"items": ids, "comps": out_comps, "errors": errors})) +} + +/// A Photoshop layer by index or name (pixel layers only). +fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, String)> { + let l = match sel { + Value::Number(n) => d.layers.get(n.as_u64()? as usize)?, + Value::String(name) => d.layers.iter().find(|l| &l.name == name)?, + _ => return None, + }; + Some((l.index, l.name.clone())) +} + +/// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings). +fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into()); + let rate = effectcraft_time::FrameRate::FPS_29_97; + let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 }; + let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs)); + let r = s.edit("Import", None, |proj, st| { + let r = crate::psd_import::import(proj, &psd, path, &name, retain, rate, duration); + st.project_selection = vec![r.comp]; + Ok(r) + })?; + let mut items = vec![r.folder.0]; + items.extend(r.items.iter().map(|i| i.0)); + Ok((r.comp.0, items, r.warnings)) } fn project_settings(s: &mut Session, p: &Value) -> Result { @@ -224,7 +308,15 @@ pub fn specs() -> Vec { cmd!("file.saveAs", "Save As...", ["File", "Save As"], Some("Cmd+Shift+S"), "{path}", always, save_as), cmd!("file.incrementAndSave", "Increment and Save", ["File"], Some("Cmd+Alt+Shift+S"), "{}", has_path, increment_save), cmd!("file.revert", "Revert", ["File"], None, "{}", has_path, revert), - cmd!("file.import", "File...", ["File", "Import"], Some("Cmd+I"), "{paths: [string]}", always, import), + cmd!( + "file.import", + "File...", + ["File", "Import"], + Some("Cmd+I"), + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + always, + import + ), cmd!( "file.projectSettings", "Project Settings...", diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2dff..b62664afb 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage { missing: true, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f6..0791d1a35 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -6,6 +6,7 @@ pub mod anim_tools; mod animation; mod comp; mod comp_more; +mod create; mod edit; mod effect; mod file; @@ -118,6 +119,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(help::specs()); v.extend(query::specs()); v.extend(layer_menu::specs()); + v.extend(create::specs()); v.extend(animation::specs()); v.extend(view::specs()); v.extend(viewer_cmds::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd76..f8bca154a 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -34,7 +34,6 @@ pub fn specs() -> Vec { // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), stub!("layer.environment", "Environment Layer", ["Layer"], "{}"), - stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488ab..9d6afcc03 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -16,10 +16,12 @@ pub mod links; pub mod mask_track; pub mod menus; pub mod prefs; +pub mod psd_import; pub mod render_queue; mod session_settings; pub mod shortcuts; pub mod tracking; +pub mod vector; pub mod viewer; pub mod warp; @@ -268,6 +270,8 @@ pub struct Session { pub journal: Vec<(String, Value)>, /// Processed-layer pixels reused across frames and edits (content-keyed, never stale). pub layer_cache: Arc, + /// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled. + pub disk_cache: Option>, /// Settings (Preferences). pub prefs: prefs::Prefs, /// Bumped whenever settings change (frontends re-apply theme, labels…). @@ -311,6 +315,7 @@ impl Default for Session { events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), + disk_cache: None, prefs: prefs::Prefs::default(), prefs_revision: 0, config: None, @@ -393,6 +398,9 @@ impl Session { pub fn bump(&mut self) { self.revision += 1; + if self.disk_cache.is_some() { + self.layer_cache.set_disk_salt(effectcraft_render::disk_cache::footage_salt(&self.project)); + } self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -574,6 +582,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_disk_cache; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; @@ -602,6 +612,8 @@ mod tests_timeline; #[cfg(test)] mod tests_track; #[cfg(test)] +mod tests_vector_import; +#[cfg(test)] mod tests_viewer; /// Font families available to text layers (bundled + scanned system fonts). diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs new file mode 100644 index 000000000..c14888ea1 --- /dev/null +++ b/crates/engine/src/psd_import.rs @@ -0,0 +1,634 @@ +//! Photoshop import as a composition (File ▸ Import ▸ File… with Import As: Composition or +//! Composition – Retain Layer Sizes), and Layer ▸ Create ▸ Convert to Editable Text. +//! +//! Every pixel layer becomes a footage item that names its layer (`Footage::layer`), placed in +//! a composition the size of the document. Groups become precomps (pass-through groups collapse +//! so their blend modes act on the layers below, like Photoshop); type layers become editable +//! text layers; layer masks are baked into the layer's alpha and vector masks become masks; +//! layer effects map to Layer Styles; adjustment layers become adjustment layers with the +//! matching effect (best effort); clipped layers preserve the transparency underneath. + +use std::sync::Arc; + +use effectcraft_color::BlendMode; +use effectcraft_color::Label; +use effectcraft_keyframe::{Gradient, Justify, ShapePath, TextDoc, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::styles::{self, STYLE_BLEND_MODES}; +use effectcraft_project::{AlphaMode, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, Project, PropGroup, Solid, SourceLayer}; +use effectcraft_psd::{Adjustment, DValue, Descriptor, Node, Psd}; +use effectcraft_time::{FrameRate, Tick}; + +/// Photoshop blend mode key → compositor mode (`pass` = pass through, handled by collapsing). +pub fn blend_mode(key: &str) -> BlendMode { + match key { + "diss" => BlendMode::Dissolve, + "dark" => BlendMode::Darken, + "mul " => BlendMode::Multiply, + "idiv" => BlendMode::ColorBurn, + "lbrn" => BlendMode::LinearBurn, + "dkCl" => BlendMode::DarkerColor, + "lite" => BlendMode::Lighten, + "scrn" => BlendMode::Screen, + "div " => BlendMode::ColorDodge, + "lddg" => BlendMode::LinearDodge, + "lgCl" => BlendMode::LighterColor, + "over" => BlendMode::Overlay, + "sLit" => BlendMode::SoftLight, + "hLit" => BlendMode::HardLight, + "vLit" => BlendMode::VividLight, + "lLit" => BlendMode::LinearLight, + "pLit" => BlendMode::PinLight, + "hMix" => BlendMode::HardMix, + "diff" => BlendMode::Difference, + "smud" => BlendMode::Exclusion, + "fsub" => BlendMode::Subtract, + "fdiv" => BlendMode::Divide, + "hue " => BlendMode::Hue, + "sat " => BlendMode::Saturation, + "colr" => BlendMode::Color, + "lum " => BlendMode::Luminosity, + _ => BlendMode::Normal, + } +} + +/// Descriptor blend mode enum (`BlnM`) → compositor mode. +fn desc_blend(v: Option<&str>) -> Option { + Some(match v? { + "Nrml" => BlendMode::Normal, + "Dslv" => BlendMode::Dissolve, + "Drkn" => BlendMode::Darken, + "Mltp" => BlendMode::Multiply, + "CBrn" => BlendMode::ColorBurn, + "linearBurn" => BlendMode::LinearBurn, + "darkerColor" => BlendMode::DarkerColor, + "Lghn" => BlendMode::Lighten, + "Scrn" => BlendMode::Screen, + "CDdg" => BlendMode::ColorDodge, + "linearDodge" => BlendMode::LinearDodge, + "lighterColor" => BlendMode::LighterColor, + "Ovrl" => BlendMode::Overlay, + "SftL" => BlendMode::SoftLight, + "HrdL" => BlendMode::HardLight, + "vividLight" => BlendMode::VividLight, + "linearLight" => BlendMode::LinearLight, + "pinLight" => BlendMode::PinLight, + "hardMix" => BlendMode::HardMix, + "Dfrn" => BlendMode::Difference, + "Xclu" => BlendMode::Exclusion, + "blendSubtraction" => BlendMode::Subtract, + "blendDivide" => BlendMode::Divide, + "H " => BlendMode::Hue, + "Strt" => BlendMode::Saturation, + "Clr " => BlendMode::Color, + "Lmns" => BlendMode::Luminosity, + _ => return None, + }) +} + +/// What the import created. +pub struct PsdImport { + pub comp: ItemId, + pub folder: ItemId, + pub items: Vec, + pub warnings: Vec, +} + +struct Ctx<'a> { + psd: &'a Psd, + path: &'a str, + retain: bool, + folder: ItemId, + rate: FrameRate, + duration: Tick, + items: Vec, + warnings: Vec, +} + +/// Import `psd` (read from `path`) as a composition named `name`. +pub fn import(proj: &mut Project, psd: &Psd, path: &str, name: &str, retain: bool, rate: FrameRate, duration: Tick) -> PsdImport { + let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder); + let mut cx = Ctx { psd, path, retain, folder, rate, duration, items: vec![], warnings: vec![] }; + let comp = build_comp(proj, &mut cx, name, &psd.tree(), None); + PsdImport { comp, folder, items: cx.items, warnings: cx.warnings } +} + +fn new_comp(cx: &Ctx) -> Comp { + Comp::new(cx.psd.width, cx.psd.height, cx.rate, cx.duration) +} + +fn build_comp(proj: &mut Project, cx: &mut Ctx, name: &str, nodes: &[Node], parent: Option) -> ItemId { + let cid = proj.add_item(name, Label::Sandstone, parent, ItemKind::Comp(Arc::new(new_comp(cx)))); + cx.items.push(cid); + let mut comp = new_comp(cx); + for n in nodes { + if let Some(l) = build_layer(proj, cx, &comp, n) { + comp.layers.push(l); + } + } + styles::sync_comp(&mut comp); + if let Some(item) = proj.item_mut(cid) { + item.kind = ItemKind::Comp(Arc::new(comp)); + } + cid +} + +fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> (ItemId, (u32, u32)) { + let l = &cx.psd.layers[index]; + let retain = cx.retain; + let (w, h) = if retain && !l.rect.is_empty() { (l.rect.width(), l.rect.height()) } else { (cx.psd.width, cx.psd.height) }; + let file = std::path::Path::new(cx.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let f = Footage { + path: cx.path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: cx.rate, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PSD".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }), + }; + let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); + cx.items.push(id); + (id, (w, h)) +} + +fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option { + let (index, children) = match n { + Node::Layer(i) => (*i, None), + Node::Group { layer, children } => (*layer, Some(children)), + }; + let pl = cx.psd.layers[index].clone(); + let name = if pl.name.is_empty() { format!("Layer {}", index + 1) } else { pl.name.clone() }; + let mut layer = if let Some(children) = children { + let sub = build_comp(proj, cx, &name, children, Some(cx.folder)); + let mut l = build::layer(proj, comp, &name, LayerSource::Comp { item: sub }, (comp.width, comp.height), None); + // Pass-through groups composite their layers straight into the parent. + l.switches.collapse = pl.blend_key() == "pass"; + l + } else if let Some(t) = &pl.text { + text_layer(proj, comp, &name, t) + } else if let Some(adj) = &pl.adjustment { + let mut l = adjustment_layer(proj, comp, &name); + add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name); + l + } else if pl.has_pixels() || pl.fill.is_some() { + let (item, size) = footage_item(proj, cx, index, &name); + let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None); + if cx.retain && !pl.rect.is_empty() { + set_vec2(&mut l, "transform/position", [pl.rect.left as f64 + size.0 as f64 / 2.0, pl.rect.top as f64 + size.1 as f64 / 2.0]); + } + l + } else { + // Empty layers have nothing to show. + return None; + }; + layer.name = comp.unique_layer_name(&name); + layer.blend_mode = blend_mode(pl.blend_key()); + layer.switches.video = !pl.hidden; + layer.preserve_transparency = pl.clipping; + set_scalar(&mut layer, "transform/opacity", pl.opacity as f64 / 255.0 * 100.0); + // Vector mask → a mask in layer space. + let origin = layer_origin(&pl, cx.retain, &layer); + if let Some(vm) = pl.vector_mask.as_ref().filter(|v| !v.disabled) + && let Some(masks) = layer.props.sub_mut("masks") + { + let mut ids = Ids(&mut proj.next_id); + for (k, sp) in vm.subpaths.iter().enumerate() { + let n = sp.knots.len(); + let mut path = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: sp.closed }; + for kn in &sp.knots { + let v = [kn[1][0] - origin[0], kn[1][1] - origin[1]]; + path.vertices.push(v); + path.in_tangents.push([kn[0][0] - kn[1][0], kn[0][1] - kn[1][1]]); + path.out_tangents.push([kn[2][0] - kn[1][0], kn[2][1] - kn[1][1]]); + } + if n == 0 { + continue; + } + let mode = if vm.inverted { MaskMode::Subtract } else { MaskMode::Add }; + masks.children.push(build::mask(&mut ids, &format!("Mask {}", k + 1), path, mode, [255, 255, 0]).into()); + } + } + // Layer effects → Layer Styles; Fill Opacity → Advanced Blending. + let gl = comp.global_light.clone(); + if let Some(fx) = &pl.effects + && fx.bool("masterFXSwitch") != Some(false) + && layer.can_have_styles() + { + apply_effects(proj, &mut layer, fx, &gl); + } + if pl.fill_opacity < 255 && layer.can_have_styles() { + let mut ids = Ids(&mut proj.next_id); + if layer.props.sub(styles::GROUP).is_none() { + let g = styles::layer_styles(&mut ids, &gl); + let at = layer.props.children.iter().position(|c| c.match_id() == "transform").map(|i| i + 1).unwrap_or(layer.props.children.len()); + layer.props.children.insert(at, g.into()); + } + if let Some(p) = layer.props.prop_mut("layerStyles/blendingOptions/advancedBlending/fillOpacity") { + p.value = Value::Scalar(pl.fill_opacity as f64 / 255.0 * 100.0); + } + } + Some(layer) +} + +/// Where layer space's origin sits in document pixels. +fn layer_origin(pl: &effectcraft_psd::Layer, retain: bool, layer: &Layer) -> [f64; 2] { + if let Some(t) = &pl.text { + return [t.transform[4], t.transform[5]]; + } + if matches!(layer.source, LayerSource::Footage { .. }) && retain && !pl.rect.is_empty() { + return [pl.rect.left as f64, pl.rect.top as f64]; + } + [0.0, 0.0] +} + +fn set_scalar(l: &mut Layer, path: &str, v: f64) { + if let Some(p) = l.props.prop_mut(path) { + p.value = Value::Scalar(v); + } +} + +fn set_vec2(l: &mut Layer, path: &str, v: [f64; 2]) { + if let Some(p) = l.props.prop_mut(path) { + p.value = match p.value { + Value::Vec3(o) => Value::Vec3([v[0], v[1], o[2]]), + _ => Value::Vec2(v), + }; + } +} + +/// Split a PostScript font name into family and style (`Inter-Bold` → `Inter`, `Bold`). +fn font_names(ps: &str) -> (String, String) { + match ps.split_once('-') { + Some((fam, style)) => (fam.to_string(), style.to_string()), + None => (ps.to_string(), "Regular".to_string()), + } +} + +fn text_doc(t: &effectcraft_psd::TextInfo) -> TextDoc { + let mut d = TextDoc::default(); + d.set_text(&t.text); + let [xx, xy, yx, yy, _, _] = t.transform; + let scale = (xx * yy - xy * yx).abs().sqrt().max(1e-6); + if let Some(s) = t.style.size { + d.size = s * scale; + } + if let Some(f) = &t.style.font { + let (fam, style) = font_names(f); + d.font = fam; + d.style = style; + } + if let Some(c) = t.style.color { + d.fill = [c[0] as f32, c[1] as f32, c[2] as f32, 1.0]; + } + d.justify = match t.style.justification { + Some(1) => Justify::Right, + Some(2) => Justify::Center, + Some(3) => Justify::JustifyLastLeft, + Some(4) => Justify::JustifyLastRight, + Some(5) => Justify::JustifyLastCenter, + Some(6) => Justify::JustifyAll, + _ => Justify::Left, + }; + if let Some(tr) = t.style.tracking { + d.tracking = tr; + } + if let Some(l) = t.style.leading { + d.leading = Some(l * scale); + } + d.faux_bold = t.style.faux_bold; + d.faux_italic = t.style.faux_italic; + if t.style.box_text + && let Some([l, top, r, b]) = t.bounds + { + d.box_size = Some([(r - l) * scale, (b - top) * scale]); + d.box_pos = [l * scale, top * scale]; + } + d +} + +fn text_layer(proj: &mut Project, comp: &Comp, name: &str, t: &effectcraft_psd::TextInfo) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Text, (0, 0), None); + if let Some(p) = l.props.prop_mut("text/sourceText") { + p.value = Value::Text(Box::new(text_doc(t))); + } + set_vec2(&mut l, "transform/position", [t.transform[4], t.transform[5]]); + let [xx, xy, _, _, _, _] = t.transform; + let rot = xy.atan2(xx).to_degrees(); + if rot.abs() > 1e-6 { + set_scalar(&mut l, "transform/rotation", rot); + } + l +} + +fn adjustment_layer(proj: &mut Project, comp: &Comp, name: &str) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (comp.width, comp.height), None); + l.switches.adjustment = true; + l.label = Label::Purple; + l +} + +fn add_effect(proj: &mut Project, l: &mut Layer, effect: &str, comp: &Comp, params: &[(&str, Value)]) { + let Some(spec) = effectcraft_effects::lookup(effect) else { return }; + let mut ids = Ids(&mut proj.next_id); + let mut g = effectcraft_effects::instantiate(spec, &mut ids, spec.name, [comp.width as f64, comp.height as f64]); + for (k, v) in params { + if let Some(p) = g.get_mut(k) { + p.value = v.clone(); + } + } + if let Some(fx) = l.props.sub_mut("effects") { + fx.children.push(g.into()); + } +} + +fn add_adjustment_effect(proj: &mut Project, l: &mut Layer, adj: &Adjustment, comp: &Comp, warnings: &mut Vec, name: &str) { + match adj { + Adjustment::Invert => add_effect(proj, l, "Invert", comp, &[]), + Adjustment::BrightnessContrast { brightness, contrast, .. } => { + add_effect(proj, l, "Brightness & Contrast", comp, &[("brightness", Value::Scalar(*brightness)), ("contrast", Value::Scalar(*contrast))]) + } + Adjustment::HueSaturation { hue, saturation, lightness, colorize } => { + let mut p = vec![("hue", Value::Scalar(*hue)), ("saturation", Value::Scalar(*saturation)), ("lightness", Value::Scalar(*lightness))]; + if *colorize { + p = vec![ + ("colorize", Value::Bool(true)), + ("colorizeHue", Value::Scalar(*hue)), + ("colorizeSaturation", Value::Scalar(*saturation)), + ("colorizeLightness", Value::Scalar(*lightness)), + ]; + } + add_effect(proj, l, "Hue/Saturation", comp, &p) + } + Adjustment::Levels { in_black, in_white, gamma, out_black, out_white } => add_effect( + proj, + l, + "Levels", + comp, + &[ + ("inBlack", Value::Scalar(in_black / 255.0)), + ("inWhite", Value::Scalar(in_white / 255.0)), + ("gamma", Value::Scalar(*gamma)), + ("outBlack", Value::Scalar(out_black / 255.0)), + ("outWhite", Value::Scalar(out_white / 255.0)), + ], + ), + Adjustment::Exposure { exposure, offset, gamma } => add_effect( + proj, + l, + "Exposure", + comp, + &[("exposure", Value::Scalar(*exposure)), ("offset", Value::Scalar(*offset)), ("gamma", Value::Scalar(*gamma))], + ), + Adjustment::Other(k) => warnings.push(format!("{name}: adjustment `{k}` is not converted (layer kept as an empty adjustment layer)")), + } +} + +// ---------------------------------------------------------------- layer effects → styles + +fn style_mode(m: BlendMode) -> Value { + Value::Enum(STYLE_BLEND_MODES.iter().position(|x| x.1 == m).unwrap_or(0) as u32) +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn color4(c: [f64; 3]) -> Value { + Value::Color([c[0], c[1], c[2], 1.0]) +} + +/// Copy the common style settings (`Md `, `Clr `, `Opct`, angle, distance, size…). +fn common(g: &mut PropGroup, d: &Descriptor) { + if let Some(m) = desc_blend(d.enum_value("Md ")) { + set(g, "blendMode", style_mode(m)); + } + if let Some(c) = d.color("Clr ") { + set(g, "color", color4(c)); + } + for (k, prop) in [("Opct", "opacity"), ("Nose", "noise"), ("Dstn", "distance"), ("blur", "size"), ("lagl", "angle"), ("ShdN", "jitter"), ("Inpr", "range")] + { + if let Some(v) = d.num(k) { + set(g, prop, Value::Scalar(v)); + } + } + if let Some(b) = d.bool("uglg") { + set(g, "useGlobalLight", Value::Bool(b)); + } +} + +fn gradient_of(d: &Descriptor) -> Option { + let g = d.obj("Grad")?; + let list = |k: &str| match g.get(k) { + Some(DValue::List(v)) => v.clone(), + _ => vec![], + }; + let colors: Vec<(f64, [f32; 4])> = list("Clrs") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => { + let col = s.color("Clr ")?; + Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, [col[0] as f32, col[1] as f32, col[2] as f32, 1.0])) + } + _ => None, + }) + .collect(); + let opacities: Vec<(f64, f32)> = list("Trns") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, (s.num("Opct").unwrap_or(100.0) / 100.0) as f32)), + _ => None, + }) + .collect(); + (!colors.is_empty()).then(|| Gradient { colors, opacities: if opacities.is_empty() { vec![(0.0, 1.0), (1.0, 1.0)] } else { opacities } }) +} + +/// The effect descriptor for `key`, or the first enabled one of its `…Multi` list. +fn effect<'a>(fx: &'a Descriptor, key: &str, multi: &str) -> Option<&'a Descriptor> { + fx.obj(key).or_else(|| match fx.get(multi) { + Some(DValue::List(v)) => v.iter().find_map(|x| match x { + DValue::Descriptor(d) if d.bool("enab") != Some(false) => Some(d), + _ => None, + }), + _ => None, + }) +} + +fn apply_effects(proj: &mut Project, layer: &mut Layer, fx: &Descriptor, gl: &styles::GlobalLight) { + const MAP: &[(&str, &str, &str)] = &[ + ("DrSh", "dropShadowMulti", "dropShadow"), + ("IrSh", "innerShadowMulti", "innerShadow"), + ("OrGl", "outerGlowMulti", "outerGlow"), + ("IrGl", "innerGlowMulti", "innerGlow"), + ("ebbl", "bevelEmbossMulti", "bevelEmboss"), + ("ChFX", "satinMulti", "satin"), + ("SoFi", "solidFillMulti", "colorOverlay"), + ("GrFl", "gradientFillMulti", "gradientOverlay"), + ("FrFX", "frameFXMulti", "stroke"), + ]; + for (key, multi, id) in MAP { + let Some(d) = effect(fx, key, multi) else { continue }; + let enabled = d.bool("enab").unwrap_or(true); + let mut ids = Ids(&mut proj.next_id); + let Some(uid) = styles::add_style(layer, &mut ids, id, gl, enabled) else { continue }; + let Some(g) = layer.props.find_group_mut(uid) else { continue }; + common(g, d); + match *id { + "dropShadow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + if let Some(b) = d.bool("layerConceals") { + set(g, "knocksOut", Value::Bool(b)); + } + } + "innerShadow" | "innerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "choke", Value::Scalar(v)); + } + if d.enum_value("glwS") == Some("SrcC") { + set(g, "source", Value::Enum(0)); + } + } + "outerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + } + "bevelEmboss" => { + let style = match d.enum_value("bvlS") { + Some("OtrB") => 0, + Some("Embs") => 2, + Some("PlEb") => 3, + Some("strokeEmboss") => 4, + _ => 1, + }; + set(g, "style", Value::Enum(style)); + let tech = match d.enum_value("bvlT") { + Some("PrBL") => 1, + Some("Slmt") => 2, + _ => 0, + }; + set(g, "technique", Value::Enum(tech)); + if let Some(v) = d.num("srgR") { + set(g, "depth", Value::Scalar(v)); + } + set(g, "direction", Value::Enum(u32::from(d.enum_value("bvlD") == Some("Out ")))); + if let Some(v) = d.num("Sftn") { + set(g, "soften", Value::Scalar(v)); + } + if let Some(v) = d.num("Lald") { + set(g, "altitude", Value::Scalar(v)); + } + for (k, p) in [("hglM", "highlightMode"), ("sdwM", "shadowMode")] { + if let Some(m) = desc_blend(d.enum_value(k)) { + set(g, p, style_mode(m)); + } + } + for (k, p) in [("hglC", "highlightColor"), ("sdwC", "shadowColor")] { + if let Some(c) = d.color(k) { + set(g, p, color4(c)); + } + } + for (k, p) in [("hglO", "highlightOpacity"), ("sdwO", "shadowOpacity")] { + if let Some(v) = d.num(k) { + set(g, p, Value::Scalar(v)); + } + } + } + "satin" => { + if let Some(b) = d.bool("Invr") { + set(g, "invert", Value::Bool(b)); + } + } + "gradientOverlay" => { + if let Some(gr) = gradient_of(d) { + set(g, "colors", Value::Gradient(gr)); + } + if let Some(a) = d.num("Angl") { + set(g, "angle", Value::Scalar(a)); + } + let st = match d.enum_value("Type") { + Some("Rdl ") => 1, + Some("Angl") => 2, + Some("Rflc") => 3, + Some("Dmnd") => 4, + _ => 0, + }; + set(g, "style", Value::Enum(st)); + if let Some(b) = d.bool("Rvrs") { + set(g, "reverse", Value::Bool(b)); + } + if let Some(b) = d.bool("Algn") { + set(g, "alignWithLayer", Value::Bool(b)); + } + if let Some(v) = d.num("Scl ") { + set(g, "scale", Value::Scalar(v)); + } + } + "stroke" => { + if let Some(v) = d.num("Sz ") { + set(g, "size", Value::Scalar(v)); + } + let pos = match d.enum_value("Styl") { + Some("InsF") => 1, + Some("CtrF") => 2, + _ => 0, + }; + set(g, "position", Value::Enum(pos)); + } + _ => {} + } + } +} + +/// Layer ▸ Create ▸ Convert to Editable Text: a footage layer showing a Photoshop type layer +/// becomes a text layer (`None` when the layer's source has no text data). +pub fn editable_text(proj: &mut Project, comp: &Comp, layer: &Layer, psd: &Psd, index: usize) -> Option { + let t = psd.layers.get(index)?.text.as_ref()?; + let mut l = text_layer(proj, comp, &layer.name, t); + l.name = layer.name.clone(); + l.blend_mode = layer.blend_mode; + l.switches.video = layer.switches.video; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.start_time = layer.start_time; + if let (Some(o), Some(p)) = (layer.props.prop("transform/opacity"), l.props.prop_mut("transform/opacity")) { + *p = effectcraft_project::Property { uid: p.uid, ..o.clone() }; + } + Some(l) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blend_keys() { + assert_eq!(blend_mode("mul "), BlendMode::Multiply); + assert_eq!(blend_mode("smud"), BlendMode::Exclusion); + assert_eq!(blend_mode("pass"), BlendMode::Normal); + assert_eq!(desc_blend(Some("linearDodge")), Some(BlendMode::LinearDodge)); + assert_eq!(font_names("Inter-Bold"), ("Inter".into(), "Bold".into())); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index b0512a6a2..e54eb93bb 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -27,6 +27,7 @@ impl Session { self.prefs.normalize(); self.layer_cache.set_budget(self.prefs.layer_cache_bytes()); self.footage.set_cache_budget(self.prefs.media_cache_bytes()); + self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; if self.history.undo.len() > levels { @@ -36,6 +37,42 @@ impl Session { self.prefs_revision += 1; } + /// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache + /// and attach it to the layer cache. Without a folder setting the platform cache folder is + /// used, but only by frontends with a settings store (headless sessions stay off disk). + pub fn configure_disk_cache(&mut self) { + use effectcraft_render::disk_cache::{DiskCache, default_folder, footage_salt}; + let d = &self.prefs.disk; + let folder = if d.disk_cache_folder.trim().is_empty() { + self.config.is_some().then(default_folder) + } else { + Some(std::path::PathBuf::from(d.disk_cache_folder.trim())) + }; + let max = d.disk_cache_max_gb.max(1) as u64 * (1 << 30); + let Some(folder) = folder.filter(|_| d.disk_cache_enabled) else { + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + }; + let dc = match &self.disk_cache { + Some(c) if c.folder() == folder.as_path() => { + c.set_max_bytes(max); + c.clone() + } + _ => match DiskCache::open(&folder, max) { + Ok(c) => c, + Err(e) => { + log::warn!("disk cache at {}: {e}", folder.display()); + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + } + }, + }; + self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + self.disk_cache = Some(dc); + } + /// Write settings to the config store. pub fn save_prefs(&self) { if let Some(c) = &self.config diff --git a/crates/engine/src/tests_disk_cache.rs b/crates/engine/src/tests_disk_cache.rs new file mode 100644 index 000000000..65180ba8a --- /dev/null +++ b/crates/engine/src/tests_disk_cache.rs @@ -0,0 +1,121 @@ +//! The persistent disk cache (PRV-2) through the session: settings, layer cache persistence +//! across restarts, invalidation on edits, purging. + +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn folder(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-session-disk-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d.to_string_lossy().to_string() +} + +fn session(dir: &str) -> Session { + let mut s = Session::default(); + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir, "disk.diskCacheEnabled": true}})).unwrap(); + // Write every layer buffer, however quick to render. + s.disk_cache.as_ref().unwrap().set_min_layer_ms(0); + s +} + +fn project(s: &mut Session) -> effectcraft_project::ItemId { + s.execute("comp.new", json!({"name": "D", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap(); + s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap(); + s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap(); + s.active_comp_id().unwrap() +} + +#[test] +fn disk_cache_settings_persistence_and_invalidation() { + let dir = folder("persist"); + let mut s = session(&dir); + let st = s.execute_checked("cache.diskStats", json!({})).unwrap(); + assert_eq!(st["enabled"], json!(true)); + assert_eq!(st["maxBytes"], json!(100u64 << 30)); + let cid = project(&mut s); + let t = s.time(); + let first = s.render(cid, t, RenderOpts::default()); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + let written = dc.stats().entries; + assert!(written > 0, "layer buffers written"); + // "Restart": a new session on the same folder and project renders from disk. + let saved = s.project.clone(); + drop(s); + let mut s2 = session(&dir); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + let dc2 = s2.disk_cache.clone().unwrap(); + assert_eq!(dc2.stats().entries, written); + let again = s2.render(cid, t, RenderOpts::default()); + assert!(dc2.stats().hits > 0, "{:?}", dc2.stats()); + assert_eq!(again.data, first.data, "disk-cached pixels are exact"); + // An edit changes the keys: a miss, and new entries. + let hits = dc2.stats().hits; + s2.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 20})).unwrap(); + let edited = s2.render(cid, t, RenderOpts::default()); + assert_ne!(edited.data, first.data); + assert_eq!(dc2.stats().hits, hits); + dc2.flush(); + assert!(dc2.stats().entries > written); + // Purge ▸ All Disk Cache empties it; memory purge leaves the disk alone. + let r = s2.execute_checked("edit.purge", json!({"what": "memory"})).unwrap(); + assert_eq!(r["diskEntries"], json!(0)); + assert!(dc2.stats().entries > 0); + let r = s2.execute_checked("edit.purge", json!({"what": "disk"})).unwrap(); + assert!(r["diskEntries"].as_u64().unwrap() > 0); + assert_eq!(dc2.stats().entries, 0); + // Disabling detaches it. + s2.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap(); + assert!(s2.disk_cache.is_none()); + assert!(s2.layer_cache.disk().is_none()); +} + +#[test] +fn footage_changes_change_the_salt() { + let dir = folder("salt"); + let mut s = session(&dir); + let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id())); + std::fs::write(&foot, b"one").unwrap(); + s.edit("add", None, |p, _| { + let f = effectcraft_project::Footage { + path: foot.to_string_lossy().to_string(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + pixel_aspect: 1.0, + frame_rate: effectcraft_time::FrameRate::FPS_30, + native_rate: None, + duration: effectcraft_time::Tick::ZERO, + has_video: true, + has_audio: false, + alpha: effectcraft_project::AlphaMode::Ignore, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PNG".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: None, + }; + p.add_item("f.png", effectcraft_color::Label::Lavender, None, effectcraft_project::ItemKind::Footage(f)); + Ok(()) + }) + .unwrap(); + let a = effectcraft_render::disk_cache::footage_salt(&s.project); + std::fs::write(&foot, b"two, longer").unwrap(); + let b = effectcraft_render::disk_cache::footage_salt(&s.project); + assert_ne!(a, b); + let cid = project(&mut s); + let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap(); + let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + assert_ne!(k1, k2); + // Unrelated comps do not change it. + s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap(); + assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858caa..17ffda7ff 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -257,6 +257,7 @@ impl Importer for MockImporter { missing: false, sequence: vec![], color_profile: None, + layer: None, }) } } diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index adcf7df0f..8cf020f9d 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); @@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); diff --git a/crates/engine/src/tests_vector_import.rs b/crates/engine/src/tests_vector_import.rs new file mode 100644 index 000000000..d205630c8 --- /dev/null +++ b/crates/engine/src/tests_vector_import.rs @@ -0,0 +1,237 @@ +//! Photoshop import as a composition (IO-2) and Layer ▸ Create text conversions (SHP-5). + +use effectcraft_color::BlendMode; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{ItemKind, LayerSource, MaskMode}; +use effectcraft_psd::write::*; +use effectcraft_psd::{DValue, Descriptor, Rect}; +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-psd-tests-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +/// A layered document exercising groups, masks, blend modes, text, styles and adjustments. +pub(crate) fn sample_psd() -> Vec { + let mut d = WDoc::new(120, 80); + let shadow = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 0.5]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("uglg", DValue::Bool(false)) + .with("lagl", DValue::UnitFloat("#Ang".into(), 45.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 9.0)) + .with("blur", DValue::UnitFloat("#Pxl".into(), 4.0)), + ), + ); + let stroke = Descriptor::new("null").with( + "FrFX", + DValue::Descriptor( + Descriptor::new("FrFX") + .with("enab", DValue::Bool(true)) + .with("Styl", DValue::Enum("FStl".into(), "InsF".into())) + .with("Md ", DValue::Enum("BlnM".into(), "Nrml".into())) + .with("Opct", DValue::UnitFloat("#Prc".into(), 100.0)) + .with("Sz ", DValue::UnitFloat("#Pxl".into(), 2.0)) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 1.0, 0.0]))), + ), + ); + let mut masked = WLayer::solid("Masked Square", Rect::new(10, 10, 40, 40), [0.0, 0.8, 0.2, 1.0]).with_blend(b"mul "); + masked.mask = Some(WMask { + rect: Rect::new(10, 10, 40, 40), + data: (0..1600).map(|i| if i % 40 < 20 { 1.0 } else { 0.0 }).collect(), + default_color: 0, + disabled: false, + }); + let mut hidden = WLayer::solid("Hidden", Rect::new(0, 0, 10, 10), [1.0, 0.0, 1.0, 1.0]); + hidden.hidden = true; + let mut clipped = WLayer::solid("Clipped", Rect::new(60, 30, 20, 20), [1.0, 1.0, 1.0, 1.0]).with_blend(b"scrn").with_opacity(128); + clipped.clipping = true; + let tri = vec![[[70.0, 10.0]; 3], [[110.0, 10.0]; 3], [[90.0, 40.0]; 3]]; + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 120, 80), [0.2, 0.3, 0.4, 1.0]), + WLayer::group_end(), + masked, + hidden, + WLayer::group("Group A", true, *b"pass"), + WLayer::solid("Base", Rect::new(55, 25, 30, 30), [0.9, 0.1, 0.1, 1.0]).with_block(effects_block(&shadow)), + clipped, + WLayer::solid("Vector Masked", Rect::new(60, 0, 60, 50), [0.1, 0.1, 0.9, 1.0]) + .with_block(vector_mask_block(&[tri], 120, 80, false)) + .with_block(effects_block(&stroke)), + WLayer::empty("Hue Shift", vec![hue_sat_block(40, 10, -5)]), + WLayer::empty("Title", vec![text_block("Hello PSD", [8.0, 70.0], "Inter-Bold", 14.0, [1.0, 1.0, 1.0], 0)]), + ]; + write(&d) +} + +#[test] +fn psd_imports_as_composition() { + let path = tmp("sample.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + assert_eq!(s.state.active_comp, Some(cid)); + assert_eq!(s.project.item(cid).unwrap().name, "sample"); + let c = s.project.comp(cid).unwrap().clone(); + assert_eq!((c.width, c.height), (120, 80)); + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Title", "Hue Shift", "Vector Masked", "Clipped", "Base", "Group A", "Background"]); + // Text layer: editable text, positioned at the type origin. + let title = &c.layers[0]; + assert!(matches!(title.source, LayerSource::Text)); + let Some(KV::Text(doc)) = title.props.prop("text/sourceText").map(|p| p.value.clone()) else { panic!("text") }; + assert_eq!(doc.text, "Hello PSD"); + assert_eq!(doc.size, 14.0); + assert_eq!(doc.font, "Inter"); + assert_eq!(doc.style, "Bold"); + assert_eq!(title.props.prop("transform/position").unwrap().value.as_vec2(), [8.0, 70.0]); + // Adjustment layer with Hue/Saturation. + let hue = &c.layers[1]; + assert!(hue.switches.adjustment); + let fx = hue.effects().unwrap().groups().next().unwrap(); + assert_eq!(fx.get("hue").unwrap().value.as_f64(), 40.0); + assert_eq!(fx.get("saturation").unwrap().value.as_f64(), 10.0); + // Vector mask → mask; Stroke style. + let vm = &c.layers[2]; + let m = vm.masks().unwrap().groups().next().unwrap(); + assert!(matches!(m.kind, effectcraft_project::GroupKind::Mask { mode: MaskMode::Add, .. })); + let Some(KV::Path(p)) = m.get("path").map(|p| p.value.clone()) else { panic!("mask path") }; + assert_eq!(p.vertices.len(), 3); + assert!((p.vertices[1][0] - 110.0).abs() < 1e-3 && (p.vertices[2][1] - 40.0).abs() < 1e-3); + let st = vm.props.group("layerStyles/stroke").unwrap(); + assert_eq!(st.get("size").unwrap().value.as_f64(), 2.0); + assert_eq!(st.get("position").unwrap().value, KV::Enum(1)); + // Clipped → preserve transparency; opacity and blend mode. + let clipped = &c.layers[3]; + assert!(clipped.preserve_transparency); + assert_eq!(clipped.blend_mode, BlendMode::Screen); + assert!((clipped.props.prop("transform/opacity").unwrap().value.as_f64() - 128.0 / 255.0 * 100.0).abs() < 1e-9); + // Drop shadow. + let ds = c.layers[4].props.group("layerStyles/dropShadow").unwrap(); + assert_eq!(ds.get("distance").unwrap().value.as_f64(), 9.0); + assert_eq!(ds.get("angle").unwrap().value.as_f64(), 45.0); + assert_eq!(ds.get("useGlobalLight").unwrap().value, KV::Bool(false)); + assert_eq!(ds.get("opacity").unwrap().value.as_f64(), 60.0); + // Group → collapsed precomp (pass through) with its layers. + let g = &c.layers[5]; + assert!(g.switches.collapse); + let LayerSource::Comp { item } = g.source else { panic!("precomp") }; + let inner = s.project.comp(item).unwrap(); + let inner_names: Vec<&str> = inner.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(inner_names, ["Hidden", "Masked Square"]); + assert!(!inner.layers[0].switches.video); + assert_eq!(inner.layers[1].blend_mode, BlendMode::Multiply); + // Footage items name their Photoshop layer; document-sized footage. + let LayerSource::Footage { item } = inner.layers[1].source else { panic!("footage") }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + let sl = f.layer.as_ref().unwrap(); + assert_eq!((sl.name.as_str(), sl.layer_size), ("Masked Square", false)); + assert_eq!((f.width, f.height), (120, 80)); + assert_eq!(s.project.item(item).unwrap().name, "Masked Square/sample.psd"); + // One undo step removes everything. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.comp(cid).is_none()); +} + +#[test] +fn psd_retain_layer_sizes() { + let path = tmp("retain.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "compositionLayerSizes"})).unwrap(); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + let c = s.project.comp(cid).unwrap(); + let base = c.layer_by_name("Base").unwrap(); + let LayerSource::Footage { item } = base.source else { panic!() }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + assert_eq!((f.width, f.height), (30, 30)); + assert!(f.layer.as_ref().unwrap().layer_size); + assert_eq!(base.props.prop("transform/position").unwrap().value.as_vec2(), [70.0, 40.0]); + assert_eq!(base.props.prop("transform/anchor").unwrap().value.as_vec2(), [15.0, 15.0]); + // Vector mask vertices are in the layer's own space. + let vm = c.layer_by_name("Vector Masked").unwrap(); + let Some(KV::Path(p)) = vm.masks().unwrap().groups().next().unwrap().get("path").map(|p| p.value.clone()) else { panic!() }; + assert!((p.vertices[0][0] - 10.0).abs() < 1e-3, "{:?}", p.vertices); +} + +#[test] +fn import_rejects_unknown_kind() { + let mut s = Session::default(); + assert!(s.execute_checked("file.import", json!({"paths": ["x.psd"], "importAs": "banana"})).is_err()); +} + +fn text_session() -> (Session, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 120, "frameRate": 30, "duration": 2})).unwrap(); + let r = s.execute_checked("layer.newText", json!({"text": "Ab8", "size": 64, "position": [160, 80], "fill": "#ffcc00"})).unwrap(); + (s, r["layer"].as_u64().unwrap()) +} + +fn alpha_diff(a: &effectcraft_raster::Image, b: &effectcraft_raster::Image) -> (f32, f32) { + let mut sum = 0.0; + let mut mass = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + sum += (p[3] - q[3]).abs(); + mass += p[3]; + } + (sum / a.data.len() as f32, mass) +} + +#[test] +fn create_shapes_from_text_matches_text_render() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let opts = RenderOpts::default(); + let text_img = s.render(cid, s.time(), opts); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromText"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(matches!(c.layers[0].source, LayerSource::Shape)); + assert_eq!(c.layers[0].name, "Ab8 Outlines"); + assert!(!c.layer(effectcraft_project::LayerId(lid)).unwrap().switches.video, "source hidden"); + let groups: Vec = c.layers[0].props.sub("contents").unwrap().groups().map(|g| g.name.clone()).collect(); + assert_eq!(groups, ["A", "b", "8"]); + let shape_img = s.render(cid, s.time(), opts); + let (d, mass) = alpha_diff(&text_img, &shape_img); + assert!(mass > 500.0, "text drew something"); + assert!(d < 0.01, "mean alpha difference {d}"); + // Fill colour carried over. + let i = shape_img.data.iter().position(|p| p[3] > 0.99).unwrap(); + let p = shape_img.data[i]; + assert!((p[0] - 1.0).abs() < 0.02 && (p[1] - 0.8).abs() < 0.02 && p[2] < 0.02, "{p:?}"); +} + +#[test] +fn create_masks_from_text() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let text_img = s.render(cid, s.time(), RenderOpts::default()); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "masksFromText"})).unwrap(); + let mid = effectcraft_project::LayerId(r["layers"][0].as_u64().unwrap()); + let c = s.active_comp().unwrap(); + let l = c.layer(mid).unwrap(); + assert!(matches!(l.source, LayerSource::Solid { .. })); + // "A", "b" and "8" have 2, 2 and 3 contours. + assert_eq!(l.masks().unwrap().groups().count(), 7); + let img = s.render(cid, s.time(), RenderOpts::default()); + let (d, _) = alpha_diff(&text_img, &img); + assert!(d < 0.01, "mean alpha difference {d}"); + // Other ops need the right layer type. + assert!(s.execute_checked("layer.create", json!({"op": "shapesFromVector", "layers": [lid]})).is_err()); + assert!(s.execute_checked("layer.create", json!({"op": "nullTracing", "layers": [lid]})).is_err()); +} diff --git a/crates/engine/src/vector.rs b/crates/engine/src/vector.rs new file mode 100644 index 000000000..a3f3e040a --- /dev/null +++ b/crates/engine/src/vector.rs @@ -0,0 +1,250 @@ +//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage → +//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines). + +use effectcraft_keyframe::{Gradient, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, Layer, LayerSource, MaskMode, Project, PropGroup}; +use effectcraft_svg::{Affine, Doc, Geom, GradientKind, Node, Paint}; +use kurbo::{BezPath, Point}; + +fn shape_paths(ids: &mut Ids, path: &BezPath) -> Vec { + effectcraft_path::from_kurbo(path) + .into_iter() + .enumerate() + .map(|(i, sp)| { + let mut g = build::shape_path(ids, sp); + g.name = format!("Path {}", i + 1); + g + }) + .collect() +} + +fn det_scale(m: Affine) -> f64 { + let c = m.as_coeffs(); + (c[0] * c[3] - c[1] * c[2]).abs().sqrt() +} + +/// Axis-aligned scale + translation only (basic shapes stay parametric). +fn axis_aligned(m: Affine) -> Option<(f64, f64, f64, f64)> { + let c = m.as_coeffs(); + (c[1].abs() < 1e-9 && c[2].abs() < 1e-9 && c[0] > 0.0 && c[3] > 0.0).then_some((c[0], c[3], c[4], c[5])) +} + +fn gradient_value(g: &effectcraft_svg::Gradient) -> Gradient { + Gradient { + colors: g.stops.iter().map(|(o, c)| (*o, [c[0] as f32, c[1] as f32, c[2] as f32, 1.0])).collect(), + opacities: g.stops.iter().map(|(o, c)| (*o, c[3] as f32)).collect(), + } +} + +/// Gradient start/end points in shape-layer space (`m`: shape user space → layer space). +fn gradient_points(g: &effectcraft_svg::Gradient, m: Affine, bbox: kurbo::Rect) -> (bool, [f64; 2], [f64; 2]) { + let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY }; + let full = m * unit * g.transform; + let p = |x: f64, y: f64| { + let q = full * Point::new(x, y); + [q.x, q.y] + }; + match g.kind { + GradientKind::Linear { x1, y1, x2, y2 } => (false, p(x1, y1), p(x2, y2)), + GradientKind::Radial { cx, cy, r, .. } => (true, p(cx, cy), p(cx + r, cy)), + } +} + +fn shape_items(ids: &mut Ids, s: &effectcraft_svg::Shape, m: Affine) -> Vec { + use kurbo::Shape as _; + let mut items = vec![]; + // Geometry: parametric when the transform keeps rectangles/ellipses axis-aligned. + match (&s.geom, axis_aligned(m)) { + (Geom::Rect { x, y, w, h, rx, ry }, Some((sx, sy, tx, ty))) if (rx - ry).abs() < 1e-9 && (*rx == 0.0 || (sx - sy).abs() < 1e-9) => { + let (w2, h2) = (w * sx, h * sy); + let pos = [tx + (x + w / 2.0) * sx, ty + (y + h / 2.0) * sy]; + let mut g = build::shape_rect(ids, [w2, h2], pos, rx * sx); + g.name = "Rectangle Path 1".into(); + items.push(g); + } + (Geom::Ellipse { cx, cy, rx, ry }, Some((sx, sy, tx, ty))) => { + items.push(build::shape_ellipse(ids, [2.0 * rx * sx, 2.0 * ry * sy], [tx + cx * sx, ty + cy * sy])); + } + (g, _) => items.extend(shape_paths(ids, &(m * g.to_path()))), + } + let bbox = s.geom.to_path().bounding_box(); + if let Some(st) = &s.stroke { + let width = st.width * det_scale(m); + let mut g = match &st.paint { + Paint::Color(c) => build::shape_stroke(ids, [c[0], c[1], c[2], 1.0], width), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_stroke(ids, radial, a, b, gradient_value(gr), width) + } + }; + set(&mut g, "opacity", Value::Scalar(st.opacity * 100.0)); + set(&mut g, "cap", Value::Enum(st.cap as u32)); + set(&mut g, "join", Value::Enum(st.join as u32)); + set(&mut g, "miter", Value::Scalar(st.miter)); + if let Some((d, off)) = &st.dash + && let Some(dg) = g.sub_mut("dashes") + { + let k = det_scale(m); + set(dg, "dash", Value::Scalar(d.first().copied().unwrap_or(0.0) * k)); + set(dg, "gap", Value::Scalar(d.get(1).or(d.first()).copied().unwrap_or(0.0) * k)); + set(dg, "offset", Value::Scalar(off * k)); + } + items.push(g); + } + if let Some(f) = &s.fill { + let mut g = match f { + Paint::Color(c) => build::shape_fill(ids, [c[0], c[1], c[2], 1.0]), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_fill(ids, radial, a, b, gradient_value(gr)) + } + }; + set(&mut g, "opacity", Value::Scalar(s.fill_opacity * 100.0)); + set(&mut g, "rule", Value::Enum(u32::from(s.fill_rule == effectcraft_svg::FillRule::EvenOdd))); + items.push(g); + } + items +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn group(ids: &mut Ids, name: &str, items: Vec, opacity: f64) -> PropGroup { + let mut g = build::shape_group(ids, name, items); + if let Some(p) = g.prop_mut("transform/opacity") { + p.value = Value::Scalar(opacity * 100.0); + } + g +} + +/// Shape layer contents for an SVG document, in document pixels (transforms are baked into the +/// geometry; group and element opacity go to the group transforms). Top of the paint order first. +pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec { + fn walk(ids: &mut Ids, nodes: &[Node], m: Affine) -> Vec { + let mut out = vec![]; + for n in nodes.iter().rev() { + match n { + Node::Group(g) => { + let items = walk(ids, &g.children, m * g.transform); + if !items.is_empty() { + out.push(group(ids, &g.name, items, g.opacity)); + } + } + Node::Shape(s) => { + let items = shape_items(ids, s, m * s.transform); + if !items.is_empty() { + out.push(group(ids, &s.name, items, s.opacity)); + } + } + } + } + out + } + let root = &doc.root; + let items = walk(ids, &root.children, root.transform); + if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items } +} + +/// A copy of `src`'s transform group with fresh uids. +fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) { + let Some(tr) = src.props.sub("transform") else { return }; + let mut tr = tr.clone(); + let mut n = proj.next_id; + tr.reassign_uids(&mut n); + proj.next_id = n + 1; + if let Some(slot) = dst.props.children.iter_mut().find(|c| c.match_id() == "transform") { + *slot = tr.into(); + } +} + +fn like_source(dst: &mut Layer, src: &Layer) { + dst.start_time = src.start_time; + dst.in_point = src.in_point; + dst.out_point = src.out_point; + dst.stretch = src.stretch; + dst.parent = src.parent; + dst.switches.three_d = src.switches.three_d; +} + +/// Layer ▸ Create ▸ Create Shapes from Vector Layer: `doc` is the SVG shown by `src`. +pub fn shapes_from_vector(proj: &mut Project, comp: &Comp, src: &Layer, doc: &Doc) -> Layer { + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let items = svg_contents(&mut Ids(&mut proj.next_id), doc); + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(items.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + l +} + +/// Layer ▸ Create ▸ Create Shapes from Text: one group per character with its outline, fill and +/// stroke (as laid out at the comp time `ctx` was made for). +pub fn shapes_from_text(proj: &mut Project, comp: &Comp, ctx_project: &Project, cid: ItemId, src: &Layer, time: effectcraft_time::Tick) -> Option { + let ctx = effectcraft_render::EvalCtx::new(ctx_project, cid, comp, time); + let geom = effectcraft_render::text::text_geom(&ctx, src)?; + let chars: Vec = geom.doc.text.chars().filter(|c| !c.is_whitespace()).collect(); + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let mut groups = vec![]; + { + let mut ids = Ids(&mut proj.next_id); + for (i, g) in geom.glyphs.iter().enumerate() { + let path = g.path(); + if path.elements().is_empty() { + continue; + } + let mut items = shape_paths(&mut ids, &path); + if g.stroke_width > 0.0 && g.stroke[3] > 0.0 { + let c = g.stroke; + items.push(build::shape_stroke(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0], g.stroke_width)); + } + if g.apply_fill { + let c = g.fill; + let mut f = build::shape_fill(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0]); + set(&mut f, "opacity", Value::Scalar(c[3] as f64 * 100.0)); + items.push(f); + } + let name = chars.get(i).map(|c| c.to_string()).unwrap_or_else(|| format!("Glyph {}", i + 1)); + groups.push(group(&mut ids, &name, items, 1.0)); + } + } + // First character on top, as in After Effects. + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(groups.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + Some(l) +} + +/// Glyph outlines of a text layer in comp space (for Create Masks from Text). +pub fn text_outlines_in_comp(project: &Project, cid: ItemId, comp: &Comp, src: &Layer, time: effectcraft_time::Tick) -> Vec { + let ctx = effectcraft_render::EvalCtx::new(project, cid, comp, time); + let Some(geom) = effectcraft_render::text::text_geom(&ctx, src) else { return vec![] }; + let (m, _) = ctx.layer_to_comp(src); + let mut out = vec![]; + for g in &geom.glyphs { + let p = g.path(); + let p = effectcraft_path::transform(std::slice::from_ref(&p), &m).remove(0); + out.extend(effectcraft_path::from_kurbo(&p)); + } + out +} + +/// A comp-sized white solid with one mask per glyph contour (Difference, so counters stay open). +pub fn masks_layer(proj: &mut Project, comp: &Comp, name: &str, outlines: Vec, solid: ItemId) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: solid }, (comp.width, comp.height), None); + let mut ids = Ids(&mut proj.next_id); + if let Some(masks) = l.props.sub_mut("masks") { + for (i, sp) in outlines.into_iter().enumerate() { + masks.children.push(build::mask(&mut ids, &format!("Mask {}", i + 1), sp, MaskMode::Difference, [255, 255, 0]).into()); + } + } + l +} diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index ca2fb1b0c..cf63e5938 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -100,6 +100,7 @@ impl Scene { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index b43271c70..7143510a1 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -18,3 +18,7 @@ effectcraft-export = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-time = { workspace = true } diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs new file mode 100644 index 000000000..eb5be076b --- /dev/null +++ b/crates/host/tests/vector_import.rs @@ -0,0 +1,163 @@ +//! Photoshop and SVG import through the fully wired session (media decoding included): pixels +//! of imported compositions, SVG footage at any scale, and Create Shapes from Vector Layer. + +use effectcraft_engine::render::RenderOpts; +use effectcraft_project::ItemId; +use effectcraft_psd::Rect; +use effectcraft_psd::write::*; +use effectcraft_raster::Image; +use serde_json::json; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-host-vector-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +fn mean_diff(a: &Image, b: &Image) -> f32 { + assert_eq!((a.width, a.height), (b.width, b.height)); + let mut s = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + for c in 0..4 { + s += (p[c] - q[c]).abs(); + } + } + s / (a.data.len() * 4) as f32 +} + +fn psd_doc() -> WDoc { + let mut d = WDoc::new(64, 48); + let mut masked = WLayer::solid("Masked", Rect::new(30, 8, 24, 24), [0.1, 0.9, 0.3, 1.0]); + masked.mask = Some(WMask { + rect: Rect::new(30, 8, 24, 24), + data: (0..576).map(|i| if (i / 24) < 12 { 1.0 } else { 0.25 }).collect(), + default_color: 0, + disabled: false, + }); + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 64, 48), [0.2, 0.25, 0.3, 1.0]), + WLayer::solid("Half Red", Rect::new(4, 4, 30, 20), [1.0, 0.0, 0.0, 1.0]).with_opacity(128), + masked, + WLayer::group_end(), + WLayer::solid("In Group", Rect::new(10, 30, 20, 12), [0.9, 0.9, 0.1, 1.0]), + WLayer::group("Group", true, *b"norm"), + ]; + d +} + +/// Straight-alpha over of the layers as Photoshop composites them (normal blending), as the +/// merged image to compare the imported composition with. +fn reference(bytes: &[u8]) -> Vec<[f32; 4]> { + let p = effectcraft_psd::Psd::parse(bytes.to_vec()).unwrap(); + let mut acc = vec![[0.0f32; 4]; (p.width * p.height) as usize]; + for l in p.layers.iter().filter(|l| l.has_pixels() && !l.hidden) { + let px = p.layer_pixels(l.index, true).unwrap(); + let o = l.opacity as f32 / 255.0; + for (d, s) in acc.iter_mut().zip(&px.data) { + let a = s[3] * o; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (s[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +#[test] +fn psd_composition_renders_like_the_document() { + let mut doc = psd_doc(); + let bytes = write(&doc); + let merged = reference(&bytes); + doc.composite = Some(merged.clone()); + let bytes = write(&doc); + let path = tmp("layers.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + // Footage (merged image). + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + let merged_item = r["items"][0].as_u64().unwrap(); + // Composition and Composition – Retain Layer Sizes. + let mut renders = vec![]; + for kind in ["composition", "compositionLayerSizes"] { + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": kind})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + renders.push(s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default())); + } + // The merged footage in a comp of the same size. + s.execute("comp.new", json!({"name": "Merged", "width": 64, "height": 48, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": merged_item})).unwrap(); + let mcid = s.active_comp_id().unwrap(); + let merged_img = s.render(mcid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + assert!(mean_diff(&merged_img, &expected) < 2.0 / 255.0, "merged footage"); + for (i, img) in renders.iter().enumerate() { + let d = mean_diff(img, &expected); + assert!(d < 2.0 / 255.0, "import kind {i}: mean diff {d}"); + } + // A single layer as footage (Choose Layer). + let r = s.execute_checked("file.import", json!({"paths": [path], "layer": "Masked"})).unwrap(); + let it = s.project.item(ItemId(r["items"][0].as_u64().unwrap())).unwrap(); + assert_eq!(it.name, "Masked/layers.psd"); +} + +#[test] +fn psd_16bit_cmyk_composition_imports() { + let mut doc = psd_doc(); + doc.depth = 16; + doc.mode = effectcraft_psd::ColorMode::Cmyk; + doc.rle = false; + let bytes = write(&doc); + let merged = reference(&bytes); + let path = tmp("layers16.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + let d = mean_diff(&img, &expected); + assert!(d < 2.0 / 255.0, "mean diff {d}"); +} + +const SVG: &[u8] = include_bytes!("../../svg/tests/fixtures/basic-shapes.svg"); +const SVG2: &[u8] = include_bytes!("../../svg/tests/fixtures/paths-gradients.svg"); + +#[test] +fn svg_footage_and_shapes_from_vector_layer() { + for (name, bytes) in [("basic-shapes.svg", SVG), ("paths-gradients.svg", SVG2)] { + let path = tmp(name); + std::fs::write(&path, bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + let doc = effectcraft_svg::parse(bytes).unwrap(); + let (w, h) = doc.pixel_size(); + s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let foot = s.render(cid, t, RenderOpts::default()); + let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0); + assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised SVG"); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(!c.layers[1].switches.video); + let shapes = s.render(cid, t, RenderOpts::default()); + let d = mean_diff(&shapes, &direct); + assert!(d < 0.02, "{name}: shapes vs SVG mean diff {d}"); + // Scaled up 3×, the shape layer stays sharp and matches the SVG rasterised at 3×. + s.execute("prop.set", json!({"layer": sid, "path": "transform/scale", "value": [300, 300]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/position", "value": [0, 0]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/anchor", "value": [0, 0]})).unwrap(); + let big = s.render(cid, t, RenderOpts::default()); + let direct3 = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let d3 = mean_diff(&big, &direct3); + assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); + } +} diff --git a/crates/lottie/src/import.rs b/crates/lottie/src/import.rs index c87dc1879..22bc50235 100644 --- a/crates/lottie/src/import.rs +++ b/crates/lottie/src/import.rs @@ -171,6 +171,7 @@ pub fn import( missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot)); cx.items.push(item); diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 5140133c8..01c724d71 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() { missing: false, sequence: vec![], color_profile: None, + layer: None, }), ); let c = comp(&p, cid); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 899925693..6e83c9495 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,6 +14,8 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index fa5df2382..dce33d8ac 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -284,6 +284,22 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. +pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px[y * wu + x]; + *o = op.apply([s[0], s[1], s[2]], s[3].clamp(0.0, 1.0)); + } + }); + out +} + #[cfg(test)] mod tests { use super::*; @@ -376,3 +392,4 @@ mod tests { } } } + diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs new file mode 100644 index 000000000..204312a58 --- /dev/null +++ b/crates/media/src/layered.rs @@ -0,0 +1,62 @@ +//! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one +//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised). + +use effectcraft_project::{AlphaMode, Footage}; +use effectcraft_raster::Image; + +use crate::convert::{AlphaOp, straight_to_image}; +use crate::{MediaError, Result}; + +fn is_svg(path: &str, bytes: &[u8]) -> bool { + path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) +} + +/// Footage for a Photoshop or SVG file (`None` for other formats). +pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let mut f = crate::probe::still_footage(path, psd.width, psd.height, image::ImageFormat::Png, true); + f.codec = if psd.psb { "PSB".into() } else { "PSD".into() }; + f.alpha = if psd.merged_alpha || !psd.layers.is_empty() { AlphaMode::Straight } else { AlphaMode::Ignore }; + return Ok(Some(f)); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true); + f.codec = "SVG".into(); + f.alpha = AlphaMode::Straight; + return Ok(Some(f)); + } + Ok(None) +} + +/// Decode a Photoshop or SVG still (`None` for other formats). +pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let px = match &footage.layer { + Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size), + None => psd.composite(), + } + .map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + // The rasteriser produces premultiplied pixels already. + let _ = op; + return Ok(Some(img)); + } + Ok(None) +} + +/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize). +pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option { + let doc = effectcraft_svg::parse(bytes).ok()?; + let (w, h) = doc.pixel_size(); + let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384)); + Some(effectcraft_svg::rasterize(&doc, sw, sh, scale)) +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e3..246054b3b 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -10,9 +10,11 @@ //! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD, //! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency, //! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate -//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). +//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR); Photoshop documents (merged image or one layer) come +//! from `effectcraft-psd` and SVG from `effectcraft-svg` (rasterised at any scale). mod convert; +mod layered; mod pool; mod probe; @@ -44,7 +46,7 @@ impl From for MediaError { } /// File extensions recognised as stills. -pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"]; +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef7..b5d94299f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -359,6 +359,11 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } + // A layer of a layered still is keyed by its layer index and size mode. + FootageKind::Still if footage.layer.is_some() => { + let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64); + Loc { key: key(&footage.path, l), media_t: None } + } FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; @@ -437,6 +442,9 @@ impl Inner { match loc.media_t { None => { let bytes = self.read(path)?; + if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? { + return Ok(img); + } let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; Ok(dynamic_to_image(&img, op)) } @@ -470,6 +478,14 @@ impl FootageSource for MediaPool { } } + fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { + if footage.missing || !effectcraft_render::is_vector_footage(footage) { + return None; + } + let bytes = self.inner.read(&footage.path).ok()?; + crate::layered::rasterize_svg(&bytes, scale).map(Arc::new) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 4e1c56110..23432cc6b 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -33,6 +33,9 @@ pub fn probe_single(path: impl AsRef) -> Result { /// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if let Some(f) = crate::layered::probe(path, &bytes)? { + return Ok(f); + } if let Ok(fmt) = image::guess_format(&bytes) { return probe_still_bytes(path, &bytes, fmt); } @@ -72,6 +75,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - missing: false, sequence: Vec::new(), color_profile: v.and_then(|v| profile_of(&v.color)), + layer: None, } } @@ -88,7 +92,7 @@ fn profile_of(c: &filmcraft_color::ColorInfo) -> Option Footage { +pub(crate) fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage { let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase(); // OpenEXR stores premultiplied colour by convention; other formats straight alpha. let alpha = match (has_alpha, fmt) { @@ -114,6 +118,7 @@ fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: missing: false, sequence: Vec::new(), color_profile: None, + layer: None, } } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 1ed35c09d..b4854f54e 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -579,6 +579,21 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// One layer of a layered file (Photoshop) instead of its merged image. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer: Option, +} + +/// A layer of a layered still (a Photoshop document) used as footage +/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceLayer { + /// Layer record index in the file (bottom of the stack = 0). + pub index: u32, + pub name: String, + /// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size. + #[serde(default)] + pub layer_size: bool, } fn one() -> u32 { diff --git a/crates/psd/Cargo.toml b/crates/psd/Cargo.toml new file mode 100644 index 000000000..49dda6a3b --- /dev/null +++ b/crates/psd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "effectcraft-psd" +description = "Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer, from Adobe's published file format specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +miniz_oxide = "0.8" diff --git a/crates/psd/README.md b/crates/psd/README.md new file mode 100644 index 000000000..e238b9b2f --- /dev/null +++ b/crates/psd/README.md @@ -0,0 +1,17 @@ +# effectcraft-psd + +Reads Photoshop documents (`.psd`, and `.psb` large documents) for File ▸ Import (as footage, as a +composition, or as a composition with layer sizes retained), and writes a minimal subset (used to +generate test fixtures). + +Implemented from Adobe's public **Adobe Photoshop File Formats Specification** (November 2019 +edition): file header, colour mode data, image resources, layer and mask information (layer +records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`, +`lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`, +`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) +and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit; +Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes. + +Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and +justification of the first style run. No Adobe code, sample files or assets were used; test +fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)). diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs new file mode 100644 index 000000000..e121f823d --- /dev/null +++ b/crates/psd/src/descriptor.rs @@ -0,0 +1,430 @@ +//! The descriptor ("action descriptor") structure: typed key/value trees used by text layers, +//! layer effects, fill layers and newer adjustment layers (spec: "Descriptor structure"). + +use crate::read::Reader; +use crate::{Error, Result}; + +/// A descriptor: a class and a list of keyed items (in file order). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Descriptor { + pub class_name: String, + pub class_id: String, + pub items: Vec<(String, DValue)>, +} + +/// One descriptor value. +#[derive(Clone, Debug, PartialEq)] +pub enum DValue { + Descriptor(Descriptor), + List(Vec), + Double(f64), + /// Unit (`#Pxl`, `#Prc`, `#Ang`, `#Pnt`…) and value. + UnitFloat(String, f64), + UnitFloats(String, Vec), + Text(String), + /// Enumerated value: type and value ids. + Enum(String, String), + Integer(i32), + LargeInteger(i64), + Bool(bool), + Class(String, String), + Raw(Vec), + Reference(Vec), +} + +/// An item of a reference (`obj `). +#[derive(Clone, Debug, PartialEq)] +pub enum RefItem { + Property(String, String), + Class(String), + Enum(String, String, String), + Offset(String, i32), + Identifier(i32), + Index(i32), + Name(String, String), +} + +impl Descriptor { + pub fn get(&self, key: &str) -> Option<&DValue> { + self.items.iter().find(|(k, _)| k == key).map(|(_, v)| v) + } + pub fn obj(&self, key: &str) -> Option<&Descriptor> { + match self.get(key)? { + DValue::Descriptor(d) => Some(d), + _ => None, + } + } + /// A number: doubles, unit floats, integers. + pub fn num(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Double(v) | DValue::UnitFloat(_, v) => Some(*v), + DValue::Integer(v) => Some(*v as f64), + DValue::LargeInteger(v) => Some(*v as f64), + _ => None, + } + } + pub fn bool(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Bool(b) => Some(*b), + _ => None, + } + } + pub fn text(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Text(s) => Some(s), + _ => None, + } + } + /// The value id of an enumerated item. + pub fn enum_value(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Enum(_, v) => Some(v), + _ => None, + } + } + pub fn raw(&self, key: &str) -> Option<&[u8]> { + match self.get(key)? { + DValue::Raw(v) => Some(v), + _ => None, + } + } + /// An RGB colour object (`RGBC` with `Rd `/`Grn `/`Bl ` in 0..255) as 0..1 values; also + /// reads grayscale (`Gry `) colours. + pub fn color(&self, key: &str) -> Option<[f64; 3]> { + let c = self.obj(key)?; + if let (Some(r), Some(g), Some(b)) = (c.num("Rd "), c.num("Grn "), c.num("Bl ")) { + return Some([r / 255.0, g / 255.0, b / 255.0]); + } + if let (Some(r), Some(g), Some(b)) = (c.num("redFloat"), c.num("greenFloat"), c.num("blueFloat")) { + return Some([r, g, b]); + } + c.num("Gry ").map(|g| { + let v = 1.0 - g / 100.0; + [v, v, v] + }) + } +} + +/// A key or class id: length-prefixed string, or a 4-character code when the length is 0. +pub(crate) fn read_id(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let n = if n == 0 { 4 } else { n }; + let b = r.bytes(n)?; + Ok(String::from_utf8_lossy(b).into_owned()) +} + +pub(crate) fn read_unicode(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(r.u16()?); + } + while v.last() == Some(&0) { + v.pop(); + } + Ok(String::from_utf16_lossy(&v)) +} + +/// Read a descriptor (after its version field). +pub fn read_descriptor(r: &mut Reader) -> Result { + let class_name = read_unicode(r)?; + let class_id = read_id(r)?; + let n = r.u32()? as usize; + if n > 100_000 { + return Err(Error::Invalid("descriptor item count".into())); + } + let mut items = Vec::with_capacity(n.min(64)); + for _ in 0..n { + let key = read_id(r)?; + let ty = r.tag()?; + let v = read_value(r, &ty, 0)?; + items.push((key, v)); + } + Ok(Descriptor { class_name, class_id, items }) +} + +fn read_value(r: &mut Reader, ty: &[u8; 4], depth: usize) -> Result { + if depth > 64 { + return Err(Error::Invalid("descriptor nesting".into())); + } + Ok(match ty { + b"Objc" | b"GlbO" => DValue::Descriptor(read_descriptor(r)?), + b"VlLs" => { + let n = r.u32()? as usize; + if n > 1_000_000 { + return Err(Error::Invalid("descriptor list length".into())); + } + let mut v = Vec::with_capacity(n.min(256)); + for _ in 0..n { + let t = r.tag()?; + v.push(read_value(r, &t, depth + 1)?); + } + DValue::List(v) + } + b"doub" => DValue::Double(r.f64()?), + b"UntF" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + DValue::UnitFloat(unit, r.f64()?) + } + b"UnFl" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n.min(1024)); + for _ in 0..n { + v.push(r.f64()?); + } + DValue::UnitFloats(unit, v) + } + b"TEXT" => DValue::Text(read_unicode(r)?), + b"enum" => { + let t = read_id(r)?; + DValue::Enum(t, read_id(r)?) + } + b"long" => DValue::Integer(r.u32()? as i32), + b"comp" => DValue::LargeInteger(r.u64()? as i64), + b"bool" => DValue::Bool(r.u8()? != 0), + b"type" | b"GlbC" => { + let name = read_unicode(r)?; + DValue::Class(name, read_id(r)?) + } + b"alis" | b"tdta" | b"Pth " => { + let n = r.u32()? as usize; + DValue::Raw(r.bytes(n)?.to_vec()) + } + b"obj " => { + let n = r.u32()? as usize; + let mut items = Vec::with_capacity(n.min(16)); + for _ in 0..n { + let t = r.tag()?; + items.push(match &t { + b"prop" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Property(class, read_id(r)?) + } + b"Clss" => { + let _name = read_unicode(r)?; + RefItem::Class(read_id(r)?) + } + b"Enmr" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + let ty = read_id(r)?; + RefItem::Enum(class, ty, read_id(r)?) + } + b"rele" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Offset(class, r.u32()? as i32) + } + b"Idnt" => RefItem::Identifier(r.u32()? as i32), + b"indx" => RefItem::Index(r.u32()? as i32), + b"name" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Name(class, read_unicode(r)?) + } + other => return Err(Error::Invalid(format!("reference item `{}`", String::from_utf8_lossy(other)))), + }); + } + DValue::Reference(items) + } + other => return Err(Error::Invalid(format!("descriptor type `{}`", String::from_utf8_lossy(other)))), + }) +} + +// ---------------------------------------------------------------- writing + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +fn put_id(out: &mut Vec, id: &str) { + if id.len() == 4 { + put_u32(out, 0); + } else { + put_u32(out, id.len() as u32); + } + out.extend_from_slice(id.as_bytes()); +} + +pub(crate) fn put_unicode(out: &mut Vec, s: &str) { + let v: Vec = s.encode_utf16().chain(std::iter::once(0)).collect(); + put_u32(out, v.len() as u32); + for c in v { + out.extend_from_slice(&c.to_be_bytes()); + } +} + +/// Serialise a descriptor (without the version field). +pub fn write_descriptor(out: &mut Vec, d: &Descriptor) { + put_unicode(out, &d.class_name); + put_id(out, &d.class_id); + put_u32(out, d.items.len() as u32); + for (k, v) in &d.items { + put_id(out, k); + write_value(out, v); + } +} + +fn write_value(out: &mut Vec, v: &DValue) { + match v { + DValue::Descriptor(d) => { + out.extend_from_slice(b"Objc"); + write_descriptor(out, d); + } + DValue::List(l) => { + out.extend_from_slice(b"VlLs"); + put_u32(out, l.len() as u32); + for x in l { + write_value(out, x); + } + } + DValue::Double(x) => { + out.extend_from_slice(b"doub"); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloat(u, x) => { + out.extend_from_slice(b"UntF"); + out.extend_from_slice(&tag4(u)); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloats(u, xs) => { + out.extend_from_slice(b"UnFl"); + out.extend_from_slice(&tag4(u)); + put_u32(out, xs.len() as u32); + for x in xs { + out.extend_from_slice(&x.to_be_bytes()); + } + } + DValue::Text(s) => { + out.extend_from_slice(b"TEXT"); + put_unicode(out, s); + } + DValue::Enum(t, e) => { + out.extend_from_slice(b"enum"); + put_id(out, t); + put_id(out, e); + } + DValue::Integer(i) => { + out.extend_from_slice(b"long"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::LargeInteger(i) => { + out.extend_from_slice(b"comp"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::Bool(b) => { + out.extend_from_slice(b"bool"); + out.push(*b as u8); + } + DValue::Class(n, c) => { + out.extend_from_slice(b"type"); + put_unicode(out, n); + put_id(out, c); + } + DValue::Raw(b) => { + out.extend_from_slice(b"tdta"); + put_u32(out, b.len() as u32); + out.extend_from_slice(b); + } + DValue::Reference(items) => { + out.extend_from_slice(b"obj "); + put_u32(out, items.len() as u32); + for it in items { + match it { + RefItem::Property(c, k) => { + out.extend_from_slice(b"prop"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, k); + } + RefItem::Class(c) => { + out.extend_from_slice(b"Clss"); + put_unicode(out, ""); + put_id(out, c); + } + RefItem::Enum(c, t, e) => { + out.extend_from_slice(b"Enmr"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, t); + put_id(out, e); + } + RefItem::Offset(c, o) => { + out.extend_from_slice(b"rele"); + put_unicode(out, ""); + put_id(out, c); + out.extend_from_slice(&o.to_be_bytes()); + } + RefItem::Identifier(i) => { + out.extend_from_slice(b"Idnt"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Index(i) => { + out.extend_from_slice(b"indx"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Name(c, n) => { + out.extend_from_slice(b"name"); + put_unicode(out, ""); + put_id(out, c); + put_unicode(out, n); + } + } + } + } + } +} + +fn tag4(s: &str) -> [u8; 4] { + let mut t = *b" "; + for (i, b) in s.bytes().take(4).enumerate() { + t[i] = b; + } + t +} + +/// Builder helpers for descriptors (tests and the writer). +impl Descriptor { + pub fn new(class_id: &str) -> Descriptor { + Descriptor { class_name: String::new(), class_id: class_id.into(), items: vec![] } + } + pub fn with(mut self, key: &str, v: DValue) -> Descriptor { + self.items.push((key.into(), v)); + self + } + /// An `RGBC` colour object from 0..1 values. + pub fn rgb(c: [f64; 3]) -> Descriptor { + Descriptor::new("RGBC").with("Rd ", DValue::Double(c[0] * 255.0)).with("Grn ", DValue::Double(c[1] * 255.0)).with("Bl ", DValue::Double(c[2] * 255.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let d = Descriptor::new("null") + .with("Txt ", DValue::Text("Héllo".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 0.5, 0.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 75.0)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("enab", DValue::Bool(true)) + .with("longKeyName", DValue::List(vec![DValue::Integer(3), DValue::Double(1.5)])) + .with("EngineData", DValue::Raw(b"<< >>".to_vec())) + .with("ref", DValue::Reference(vec![RefItem::Enum("Lyr ".into(), "Ordn".into(), "Trgt".into()), RefItem::Index(2)])); + let mut out = vec![]; + write_descriptor(&mut out, &d); + let back = read_descriptor(&mut Reader::new(&out)).unwrap(); + assert_eq!(back.class_id, "null"); + assert_eq!(back.text("Txt "), Some("Héllo")); + assert_eq!(back.color("Clr "), Some([1.0, 0.5, 0.0])); + assert_eq!(back.num("Opct"), Some(75.0)); + assert_eq!(back.enum_value("Md "), Some("Mltp")); + assert_eq!(back.bool("enab"), Some(true)); + assert_eq!(back.raw("EngineData"), Some(&b"<< >>"[..])); + assert_eq!(back, d); + } +} diff --git a/crates/psd/src/engine_data.rs b/crates/psd/src/engine_data.rs new file mode 100644 index 000000000..c4f230380 --- /dev/null +++ b/crates/psd/src/engine_data.rs @@ -0,0 +1,288 @@ +//! The text engine data of type layers (`EngineData` in the `TySh` descriptor): a PostScript-like +//! tree of dictionaries (`<< /Key value >>`), arrays (`[ ]`), numbers, names, booleans and +//! strings (`(þÿ` + UTF-16BE with backslash escapes `)`). Read best-effort. + +#[derive(Clone, Debug, PartialEq)] +pub enum EValue { + Dict(Vec<(String, EValue)>), + Array(Vec), + Num(f64), + Bool(bool), + Name(String), + Str(String), +} + +impl EValue { + pub fn get(&self, key: &str) -> Option<&EValue> { + match self { + EValue::Dict(v) => v.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + pub fn idx(&self, i: usize) -> Option<&EValue> { + match self { + EValue::Array(v) => v.get(i), + _ => None, + } + } + /// Follow a `/`-separated path; numeric segments index arrays. + pub fn path(&self, p: &str) -> Option<&EValue> { + let mut cur = self; + for seg in p.split('/').filter(|s| !s.is_empty()) { + cur = match seg.parse::() { + Ok(i) => cur.idx(i)?, + Err(_) => cur.get(seg)?, + }; + } + Some(cur) + } + pub fn num(&self) -> Option { + match self { + EValue::Num(n) => Some(*n), + _ => None, + } + } + pub fn str(&self) -> Option<&str> { + match self { + EValue::Str(s) | EValue::Name(s) => Some(s), + _ => None, + } + } +} + +struct P<'a> { + b: &'a [u8], + i: usize, +} + +impl P<'_> { + fn ws(&mut self) { + while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == 0) { + self.i += 1; + } + } + fn peek(&self, s: &[u8]) -> bool { + self.b[self.i..].starts_with(s) + } + fn value(&mut self, depth: usize) -> Option { + if depth > 128 { + return None; + } + self.ws(); + if self.i >= self.b.len() { + return None; + } + if self.peek(b"<<") { + self.i += 2; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Dict(v)); + } + if self.peek(b">>") { + self.i += 2; + return Some(EValue::Dict(v)); + } + if self.b[self.i] != b'/' { + // Unexpected token: skip it. + self.i += 1; + continue; + } + let key = self.name(); + let val = self.value(depth + 1)?; + v.push((key, val)); + } + } + match self.b[self.i] { + b'[' => { + self.i += 1; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Array(v)); + } + if self.b[self.i] == b']' { + self.i += 1; + return Some(EValue::Array(v)); + } + let start = self.i; + match self.value(depth + 1) { + Some(x) => v.push(x), + None => return Some(EValue::Array(v)), + } + if self.i == start { + self.i += 1; + } + } + } + b'/' => Some(EValue::Name(self.name())), + b'(' => Some(EValue::Str(self.string())), + _ => { + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + let tok = std::str::from_utf8(&self.b[start..self.i]).unwrap_or(""); + if self.i == start { + self.i += 1; + return Some(EValue::Name(String::new())); + } + Some(match tok { + "true" => EValue::Bool(true), + "false" => EValue::Bool(false), + t => t.parse::().map(EValue::Num).unwrap_or_else(|_| EValue::Name(t.to_string())), + }) + } + } + } + fn name(&mut self) -> String { + self.i += 1; // '/' + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + String::from_utf8_lossy(&self.b[start..self.i]).into_owned() + } + fn string(&mut self) -> String { + self.i += 1; // '(' + let mut raw = vec![]; + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'\\' if self.i < self.b.len() => { + let e = self.b[self.i]; + self.i += 1; + raw.push(match e { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + other => other, + }); + } + b')' => break, + _ => raw.push(c), + } + } + if raw.starts_with(&[0xFE, 0xFF]) { + let u: Vec = raw[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u).trim_end_matches('\0').to_string() + } else { + String::from_utf8_lossy(&raw).into_owned() + } + } +} + +/// Parse engine data (returns an empty dictionary on garbage). +pub fn parse(b: &[u8]) -> EValue { + let mut p = P { b, i: 0 }; + p.value(0).unwrap_or(EValue::Dict(vec![])) +} + +/// Encode a string the way engine data stores it (UTF-16BE with a BOM, escaped). +pub fn encode_string(s: &str) -> Vec { + let mut out = vec![b'(', 0xFE, 0xFF]; + for u in s.encode_utf16() { + for b in u.to_be_bytes() { + if matches!(b, b'(' | b')' | b'\\') { + out.push(b'\\'); + } + out.push(b); + } + } + out.push(b')'); + out +} + +/// The first style run's character attributes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextStyle { + pub font: Option, + pub size: Option, + /// RGB 0..1. + pub color: Option<[f64; 3]>, + /// 0 left, 1 right, 2 centre, 3+ justified variants. + pub justification: Option, + pub tracking: Option, + pub leading: Option, + pub faux_bold: bool, + pub faux_italic: bool, + /// Paragraph (box) text rather than point text. + pub box_text: bool, +} + +pub fn text_style(ed: &EValue) -> TextStyle { + let engine = ed.get("EngineDict").unwrap_or(ed); + let resources = ed.get("ResourceDict").or_else(|| ed.get("DocumentResources")); + let run = engine.path("StyleRun/RunArray/0/StyleSheet/StyleSheetData"); + let normal = resources.and_then(|r| r.path("StyleSheetSet/0/StyleSheetData")); + let attr = |k: &str| run.and_then(|r| r.get(k)).or_else(|| normal.and_then(|n| n.get(k))); + let font = + attr("Font").and_then(EValue::num).and_then(|i| resources?.path(&format!("FontSet/{}/Name", i as usize))).and_then(EValue::str).map(str::to_string); + let color = attr("FillColor").and_then(|c| c.get("Values")).and_then(|v| match v { + EValue::Array(a) if a.len() >= 4 => Some([a[1].num()?, a[2].num()?, a[3].num()?]), + _ => None, + }); + let para = engine.path("ParagraphRun/RunArray/0/ParagraphSheet/Properties"); + let para_normal = resources.and_then(|r| r.path("ParagraphSheetSet/0/Properties")); + let justification = + para.and_then(|p| p.get("Justification")).or_else(|| para_normal.and_then(|p| p.get("Justification"))).and_then(EValue::num).map(|n| n as u8); + let auto_leading = attr("AutoLeading").is_none_or(|v| *v != EValue::Bool(false)); + let box_text = engine.path("Rendered/Shapes/Children/0/ShapeType").and_then(EValue::num) == Some(1.0); + TextStyle { + font, + size: attr("FontSize").and_then(EValue::num), + color, + justification, + tracking: attr("Tracking").and_then(EValue::num), + leading: if auto_leading { None } else { attr("Leading").and_then(EValue::num) }, + faux_bold: attr("FauxBold") == Some(&EValue::Bool(true)), + faux_italic: attr("FauxItalic") == Some(&EValue::Bool(true)), + box_text, + } +} + +/// Minimal engine data for a type layer with one style run (fixtures, and what a writer needs). +pub fn minimal(text: &str, font: &str, size: f64, color: [f64; 3], justification: u8) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(b"\n\n<<\n\t/EngineDict\n\t<<\n\t\t/Editor << /Text "); + out.extend_from_slice(&encode_string(&format!("{text}\r"))); + out.extend_from_slice(b" >>\n\t\t/ParagraphRun << /RunArray [ << /ParagraphSheet << /Properties << /Justification "); + out.extend_from_slice(justification.to_string().as_bytes()); + out.extend_from_slice(b" >> >> >> ] >>\n\t\t/StyleRun << /RunArray [ << /StyleSheet << /StyleSheetData << /Font 0 /FontSize "); + out.extend_from_slice(format!("{size:.1}").as_bytes()); + out.extend_from_slice( + format!(" /FillColor << /Type 1 /Values [ 1.0 {:.5} {:.5} {:.5} ] >> /AutoLeading true >> >> >> ] >>\n\t>>\n", color[0], color[1], color[2]).as_bytes(), + ); + out.extend_from_slice(b"\t/ResourceDict << /FontSet [ << /Name "); + out.extend_from_slice(&encode_string(font)); + out.extend_from_slice(b" /Type 0 >> ] >>\n>>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_minimal() { + let b = minimal("Hi (there)", "Arial-BoldMT", 36.0, [1.0, 0.5, 0.0], 2); + let v = parse(&b); + assert_eq!(v.path("EngineDict/Editor/Text").and_then(EValue::str), Some("Hi (there)\r")); + let st = text_style(&v); + assert_eq!(st.font.as_deref(), Some("Arial-BoldMT")); + assert_eq!(st.size, Some(36.0)); + assert_eq!(st.color, Some([1.0, 0.5, 0.0])); + assert_eq!(st.justification, Some(2)); + assert!(!st.box_text); + } + + #[test] + fn garbage_is_harmless() { + let _ = parse(b"<< /A [ 1 2 << /B (unterminated"); + let _ = parse(b"]]]>>>> /x"); + let _ = parse(&[0xFF; 64]); + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs new file mode 100644 index 000000000..885ab3c21 --- /dev/null +++ b/crates/psd/src/lib.rs @@ -0,0 +1,965 @@ +//! Photoshop document (PSD / PSB) reader, implemented from Adobe's public *Photoshop File Formats +//! Specification* (see README.md), and a minimal writer ([`write`]) for fixtures. +//! +//! [`Psd::parse`] reads the structure (header, layer records and their additional information, +//! the layer tree) without decoding pixels; [`Psd::composite`] decodes the merged image and +//! [`Psd::layer_pixels`] one layer (with its layer mask applied to alpha) on demand. Pixels come +//! out as straight (unpremultiplied) RGBA `f32` in 0..1, display-encoded (32-bit documents, +//! which are linear, are encoded to sRGB). + +pub mod descriptor; +pub mod engine_data; +mod read; +pub mod write; + +use std::sync::Arc; + +pub use descriptor::{DValue, Descriptor}; +use read::Reader; + +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + NotPsd, + Truncated, + Unsupported(String), + Invalid(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotPsd => write!(f, "not a Photoshop document"), + Error::Truncated => write!(f, "truncated Photoshop document"), + Error::Unsupported(s) => write!(f, "unsupported Photoshop feature: {s}"), + Error::Invalid(s) => write!(f, "invalid Photoshop document: {s}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `bytes` start like a Photoshop document. +pub fn is_psd(bytes: &[u8]) -> bool { + bytes.len() >= 6 && &bytes[..4] == b"8BPS" && matches!(u16::from_be_bytes([bytes[4], bytes[5]]), 1 | 2) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColorMode { + Bitmap, + Grayscale, + Indexed, + Rgb, + Cmyk, + Multichannel, + Duotone, + Lab, +} + +impl ColorMode { + pub fn from_u16(v: u16) -> Option { + Some(match v { + 0 => ColorMode::Bitmap, + 1 => ColorMode::Grayscale, + 2 => ColorMode::Indexed, + 3 => ColorMode::Rgb, + 4 => ColorMode::Cmyk, + 7 => ColorMode::Multichannel, + 8 => ColorMode::Duotone, + 9 => ColorMode::Lab, + _ => return None, + }) + } + pub fn to_u16(self) -> u16 { + match self { + ColorMode::Bitmap => 0, + ColorMode::Grayscale => 1, + ColorMode::Indexed => 2, + ColorMode::Rgb => 3, + ColorMode::Cmyk => 4, + ColorMode::Multichannel => 7, + ColorMode::Duotone => 8, + ColorMode::Lab => 9, + } + } + /// Number of colour channels (before alpha). + pub fn color_channels(self) -> usize { + match self { + ColorMode::Rgb | ColorMode::Lab => 3, + ColorMode::Cmyk => 4, + _ => 1, + } + } +} + +/// A rectangle in document pixels (`right`/`bottom` exclusive). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Rect { + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Rect { + pub fn new(left: i32, top: i32, width: i32, height: i32) -> Rect { + Rect { top, left, bottom: top + height, right: left + width } + } + pub fn width(&self) -> u32 { + (self.right - self.left).max(0) as u32 + } + pub fn height(&self) -> u32 { + (self.bottom - self.top).max(0) as u32 + } + pub fn is_empty(&self) -> bool { + self.width() == 0 || self.height() == 0 + } +} + +/// One channel of a layer: id (0.. colour, -1 transparency, -2 layer mask, -3 real user mask) and +/// where its data (compression + image data) sits in the file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChannelInfo { + pub id: i16, + pub offset: usize, + pub len: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LayerMask { + pub rect: Rect, + /// Value outside the mask rectangle (0 or 255). + pub default_color: u8, + pub disabled: bool, + pub inverted: bool, +} + +/// Layer kind from the section divider setting (`lsct`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Section { + #[default] + Layer, + OpenFolder, + ClosedFolder, + /// The hidden "" record that closes a group (below its children). + Divider, +} + +/// A type layer (`TySh`). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextInfo { + /// Text with `\n` line breaks. + pub text: String, + /// Affine transform (xx, xy, yx, yy, tx, ty): the text origin sits at (tx, ty). + pub transform: [f64; 6], + pub style: engine_data::TextStyle, + /// Text bounds (left, top, right, bottom) relative to the origin, when stored. + pub bounds: Option<[f64; 4]>, +} + +/// Adjustment layers (best effort). +#[derive(Clone, Debug, PartialEq)] +pub enum Adjustment { + Invert, + BrightnessContrast { + brightness: f64, + contrast: f64, + legacy: bool, + }, + HueSaturation { + hue: f64, + saturation: f64, + lightness: f64, + colorize: bool, + }, + Levels { + in_black: f64, + in_white: f64, + gamma: f64, + out_black: f64, + out_white: f64, + }, + Exposure { + exposure: f64, + offset: f64, + gamma: f64, + }, + /// Another adjustment (its tagged-block key, e.g. `curv`, `blnc`). + Other(String), +} + +/// A vector mask path: sub-paths of (in tangent, vertex, out tangent) points in document pixels. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct VectorMask { + pub inverted: bool, + pub disabled: bool, + pub subpaths: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SubPath { + pub closed: bool, + /// Per knot: [control point preceding, anchor, control point leaving] as (x, y). + pub knots: Vec<[[f64; 2]; 3]>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Layer { + /// Index in [`Psd::layers`] (file order: bottom of the stack first). + pub index: usize, + pub name: String, + pub id: Option, + pub rect: Rect, + /// Blend mode key (`norm`, `mul `, `pass`…). + pub blend: [u8; 4], + pub opacity: u8, + /// Fill opacity (`iOpa`), 255 when absent. + pub fill_opacity: u8, + /// Clipped to the layer below. + pub clipping: bool, + pub hidden: bool, + pub protect_transparency: bool, + pub section: Section, + pub channels: Vec, + pub mask: Option, + pub text: Option, + pub adjustment: Option, + /// Solid colour fill layer (`SoCo`), RGB 0..1. + pub fill: Option<[f64; 3]>, + /// Layer effects (`lfx2` descriptor). + pub effects: Option, + pub vector_mask: Option, + /// Keys of every additional-information block the layer carries. + pub keys: Vec<[u8; 4]>, +} + +impl Layer { + pub fn is_group(&self) -> bool { + matches!(self.section, Section::OpenFolder | Section::ClosedFolder) + } + pub fn blend_key(&self) -> &str { + std::str::from_utf8(&self.blend).unwrap_or("norm") + } + pub fn has_pixels(&self) -> bool { + !self.rect.is_empty() && self.channels.iter().any(|c| c.id >= 0) + } +} + +/// The layer tree, top of the stack first. +#[derive(Clone, Debug, PartialEq)] +pub enum Node { + Layer(usize), + Group { layer: usize, children: Vec }, +} + +/// Decoded pixels: straight RGBA, row-major. +#[derive(Clone, Debug, PartialEq)] +pub struct Pixels { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, +} + +/// A parsed Photoshop document. +#[derive(Clone, Debug)] +pub struct Psd { + /// PSB (large document format). + pub psb: bool, + pub width: u32, + pub height: u32, + pub channels: u16, + pub depth: u16, + pub mode: ColorMode, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// The merged image's first extra channel is transparency. + pub merged_alpha: bool, + /// Indexed colour palette (256 R, then G, then B). + pub palette: Vec, + data: Arc<[u8]>, + merged_offset: usize, +} + +impl Psd { + pub fn parse(data: impl Into>) -> Result { + let data: Arc<[u8]> = data.into(); + let mut r = Reader::new(&data); + if r.bytes(4).ok() != Some(b"8BPS") { + return Err(Error::NotPsd); + } + let version = r.u16()?; + let psb = match version { + 1 => false, + 2 => true, + _ => return Err(Error::NotPsd), + }; + r.skip(6)?; + let channels = r.u16()?; + let height = r.u32()?; + let width = r.u32()?; + let depth = r.u16()?; + let mode_v = r.u16()?; + let mode = ColorMode::from_u16(mode_v).ok_or_else(|| Error::Unsupported(format!("colour mode {mode_v}")))?; + if !matches!(depth, 1 | 8 | 16 | 32) { + return Err(Error::Unsupported(format!("{depth}-bit"))); + } + if width == 0 || height == 0 || width > 300_000 || height > 300_000 { + return Err(Error::Invalid(format!("size {width}x{height}"))); + } + // Colour mode data. + let n = r.u32()? as usize; + let palette = r.bytes(n)?.to_vec(); + // Image resources. + let n = r.u32()? as usize; + r.skip(n)?; + // Layer and mask information. + let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 }; + let lm_len = r.length(psb)?; + let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?; + if lm_len > 0 { + let li_len = r.length(psb)?; + let li_end = r.pos.checked_add(li_len).filter(|&e| e <= lm_end).ok_or(Error::Truncated)?; + if li_len > 0 { + psd.layer_info(&data, r.pos, li_end)?; + } + r.pos = li_end; + if r.pos + 4 <= lm_end { + let g = r.u32()? as usize; + r.pos = (r.pos + g).min(lm_end); + } + // Global additional layer information (16/32-bit layer info lives here). + while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) { + if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start { + // The block holds the layer info, normally without its length field; accept + // both layouts. + if psd.layer_info(&data, start, end).is_err() || psd.layers.is_empty() { + psd.layers.clear(); + let skip = if psb { 8 } else { 4 }; + if start + skip < end && psd.layer_info(&data, start + skip, end).is_err() { + psd.layers.clear(); + } + } + } + r.pos = end; + } + } + psd.merged_offset = lm_end; + Ok(psd) + } + + fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> { + let mut r = Reader::at(&data[..end], start); + let count = r.i16()?; + self.merged_alpha = count < 0; + let n = count.unsigned_abs() as usize; + let mut layers = Vec::with_capacity(n); + for i in 0..n { + layers.push(self.layer_record(&mut r, i)?); + } + // Channel image data follows in record order. + for l in &mut layers { + for c in &mut l.channels { + c.offset = r.pos; + r.skip(c.len)?; + } + } + self.layers = layers; + Ok(()) + } + + fn layer_record(&self, r: &mut Reader, index: usize) -> Result { + let psb = self.psb; + let (top, left, bottom, right) = (r.i32()?, r.i32()?, r.i32()?, r.i32()?); + let nch = r.u16()? as usize; + if nch > 64 { + return Err(Error::Invalid("channel count".into())); + } + let mut channels = Vec::with_capacity(nch); + for _ in 0..nch { + let id = r.i16()?; + let len = r.length(psb)?; + channels.push(ChannelInfo { id, offset: 0, len }); + } + if r.bytes(4)? != b"8BIM" { + return Err(Error::Invalid("layer record signature".into())); + } + let blend = r.tag()?; + let opacity = r.u8()?; + let clipping = r.u8()? != 0; + let flags = r.u8()?; + r.skip(1)?; + let extra = r.u32()? as usize; + let extra_end = r.pos.checked_add(extra).filter(|&e| e <= r.data.len()).ok_or(Error::Truncated)?; + let mut l = Layer { + index, + rect: Rect { top, left, bottom, right }, + blend, + opacity, + fill_opacity: 255, + clipping, + hidden: flags & 2 != 0, + protect_transparency: flags & 1 != 0, + channels, + ..Default::default() + }; + // Layer mask data. + let mlen = r.u32()? as usize; + let mstart = r.pos; + if mlen >= 18 { + let rect = Rect { top: r.i32()?, left: r.i32()?, bottom: r.i32()?, right: r.i32()? }; + let default_color = r.u8()?; + let mflags = r.u8()?; + l.mask = Some(LayerMask { rect, default_color, disabled: mflags & 2 != 0, inverted: mflags & 4 != 0 }); + } + r.pos = mstart + mlen; + // Blending ranges. + let blen = r.u32()? as usize; + r.skip(blen)?; + l.name = r.pascal(4)?; + let mut pos = r.pos; + let data = r.data; + while let Some((key, start, end)) = next_block(data, &mut pos, extra_end, psb) { + l.keys.push(key); + let block = &data[start..end]; + // A malformed block only loses that block's information. + let _ = self.layer_block(&mut l, &key, block); + pos = end; + } + r.pos = extra_end; + Ok(l) + } + + fn layer_block(&self, l: &mut Layer, key: &[u8; 4], b: &[u8]) -> Result<()> { + let mut r = Reader::new(b); + match key { + b"luni" => { + let name = descriptor::read_unicode(&mut r)?; + if !name.is_empty() { + l.name = name; + } + } + b"lyid" => l.id = Some(r.u32()?), + b"iOpa" => l.fill_opacity = r.u8()?, + b"lsct" | b"lsdk" => { + l.section = match r.u32()? { + 1 => Section::OpenFolder, + 2 => Section::ClosedFolder, + 3 => Section::Divider, + _ => Section::Layer, + }; + if b.len() >= 12 && &b[4..8] == b"8BIM" { + l.blend.copy_from_slice(&b[8..12]); + } + } + b"TySh" => { + let _ver = r.u16()?; + let mut t = [0.0; 6]; + for v in &mut t { + *v = r.f64()?; + } + let _text_ver = r.u16()?; + let _desc_ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + let text = d.text("Txt ").unwrap_or_default().replace("\r\n", "\n").replace('\r', "\n"); + let text = text.strip_suffix('\n').map(str::to_string).unwrap_or(text); + let style = d.raw("EngineData").map(|e| engine_data::text_style(&engine_data::parse(e))).unwrap_or_default(); + let bounds = + d.obj("bounds").or_else(|| d.obj("boundingBox")).and_then(|o| Some([o.num("Left")?, o.num("Top ")?, o.num("Rght")?, o.num("Btom")?])); + l.text = Some(TextInfo { text, transform: t, style, bounds }); + } + b"lfx2" => { + let _ver = r.u32()?; + let _dver = r.u32()?; + l.effects = Some(descriptor::read_descriptor(&mut r)?); + } + b"SoCo" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.fill = d.color("Clr "); + } + b"nvrt" => l.adjustment = Some(Adjustment::Invert), + b"brit" => { + let brightness = r.i16()? as f64; + let contrast = r.i16()? as f64; + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::BrightnessContrast { brightness, contrast, legacy: true }); + } + } + b"CgEd" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.adjustment = Some(Adjustment::BrightnessContrast { + brightness: d.num("Brgh").unwrap_or(0.0), + contrast: d.num("Cntr").unwrap_or(0.0), + legacy: d.bool("useLegacy").unwrap_or(false), + }); + } + b"hue2" | b"hue " => { + let _ver = r.u16()?; + let colorize = r.u8()? != 0; + r.skip(1)?; + let ch = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let m = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let (hue, saturation, lightness) = if colorize { ch } else { m }; + l.adjustment = Some(Adjustment::HueSaturation { hue, saturation, lightness, colorize }); + } + b"levl" => { + let _ver = r.u16()?; + let (ib, iw, ob, ow, g) = (r.i16()?, r.i16()?, r.i16()?, r.i16()?, r.i16()?); + l.adjustment = + Some(Adjustment::Levels { in_black: ib as f64, in_white: iw as f64, gamma: g as f64 / 100.0, out_black: ob as f64, out_white: ow as f64 }); + } + b"expA" => { + let _ver = r.u16()?; + let exposure = r.f32()? as f64; + let offset = r.f32()? as f64; + let gamma = r.f32()? as f64; + l.adjustment = Some(Adjustment::Exposure { exposure, offset, gamma }); + } + b"curv" | b"blnc" | b"post" | b"thrs" | b"selc" | b"mixr" | b"phfl" | b"vibA" | b"blwh" | b"grdm" | b"clrL" => { + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned())); + } + } + b"vmsk" | b"vsms" => { + let _ver = r.u32()?; + let flags = r.u32()?; + l.vector_mask = Some(self.vector_paths(&mut r, flags)?); + } + _ => {} + } + Ok(()) + } + + fn vector_paths(&self, r: &mut Reader, flags: u32) -> Result { + let (w, h) = (self.width as f64, self.height as f64); + let mut vm = VectorMask { inverted: flags & 1 != 0, disabled: flags & 4 != 0, subpaths: vec![] }; + let fixed = |r: &mut Reader| -> Result { Ok(r.i32()? as f64 / (1 << 24) as f64) }; + while r.remaining() >= 26 { + let sel = r.u16()?; + match sel { + 0 | 3 => { + let _n = r.u16()?; + r.skip(22)?; + vm.subpaths.push(SubPath { closed: sel == 0, knots: vec![] }); + } + 1 | 2 | 4 | 5 => { + let mut pts = [[0.0; 2]; 3]; + for p in &mut pts { + let y = fixed(r)?; + let x = fixed(r)?; + *p = [x * w, y * h]; + } + if let Some(sp) = vm.subpaths.last_mut() { + sp.knots.push(pts); + } + } + _ => r.skip(24)?, + } + } + vm.subpaths.retain(|s| !s.knots.is_empty()); + Ok(vm) + } + + /// The layer tree, top of the stack first. + pub fn tree(&self) -> Vec { + let mut stack: Vec> = vec![vec![]]; + for l in &self.layers { + match l.section { + Section::Divider => stack.push(vec![]), + Section::OpenFolder | Section::ClosedFolder => { + let children = if stack.len() > 1 { stack.pop().unwrap_or_default() } else { vec![] }; + if let Some(top) = stack.last_mut() { + top.push(Node::Group { layer: l.index, children }); + } + } + Section::Layer => { + if let Some(top) = stack.last_mut() { + top.push(Node::Layer(l.index)); + } + } + } + } + while stack.len() > 1 { + let c = stack.pop().unwrap_or_default(); + if let Some(top) = stack.last_mut() { + top.extend(c); + } + } + fn rev(v: &mut Vec) { + v.reverse(); + for n in v { + if let Node::Group { children, .. } = n { + rev(children); + } + } + } + let mut out = stack.pop().unwrap_or_default(); + rev(&mut out); + out + } + + fn bytes_per_sample(&self) -> usize { + match self.depth { + 16 => 2, + 32 => 4, + _ => 1, + } + } + + /// Decode one channel of `w`×`h` samples at `offset` (compression field first). Values 0..1. + fn decode_channel(&self, offset: usize, len: usize, w: usize, h: usize) -> Result> { + let end = offset.checked_add(len).filter(|&e| e <= self.data.len()).ok_or(Error::Truncated)?; + let mut r = Reader::at(&self.data[..end], offset); + let comp = r.u16()?; + let rowb = self.row_bytes(w); + let raw = match comp { + 0 => r.bytes(rowb * h)?.to_vec(), + 1 => { + let mut counts = Vec::with_capacity(h); + for _ in 0..h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + let mut out = Vec::with_capacity(rowb * h); + for c in counts { + unpack_bits(r.bytes(c)?, rowb, &mut out)?; + } + out + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let mut v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + v.resize(rowb * h, 0); + if comp == 3 { + self.unpredict(&mut v, w, h); + } + v + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + }; + Ok(self.samples(&raw, w, h)) + } + + fn row_bytes(&self, w: usize) -> usize { + if self.depth == 1 { w.div_ceil(8) } else { w * self.bytes_per_sample() } + } + + fn unpredict(&self, v: &mut [u8], w: usize, h: usize) { + match self.depth { + 16 => { + for row in v.chunks_exact_mut(w * 2).take(h) { + let mut prev = 0u16; + for c in row.chunks_exact_mut(2) { + let x = u16::from_be_bytes([c[0], c[1]]).wrapping_add(prev); + c.copy_from_slice(&x.to_be_bytes()); + prev = x; + } + } + } + 32 => { + for row in v.chunks_exact_mut(w * 4).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + // Bytes are stored plane by plane (all first bytes, then all second bytes…). + let planar = row.to_vec(); + for x in 0..w { + for b in 0..4 { + row[x * 4 + b] = planar[b * w + x]; + } + } + } + } + _ => { + for row in v.chunks_exact_mut(w).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + } + } + } + } + + fn samples(&self, raw: &[u8], w: usize, h: usize) -> Vec { + match self.depth { + 1 => { + let rowb = w.div_ceil(8); + let mut out = Vec::with_capacity(w * h); + for y in 0..h { + for x in 0..w { + let b = raw.get(y * rowb + x / 8).copied().unwrap_or(0); + // 1 = black. + out.push(if b & (0x80 >> (x % 8)) != 0 { 0.0 } else { 1.0 }); + } + } + out + } + 16 => raw.chunks_exact(2).take(w * h).map(|c| u16::from_be_bytes([c[0], c[1]]) as f32 / 65535.0).collect(), + 32 => raw.chunks_exact(4).take(w * h).map(|c| f32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(), + _ => raw.iter().take(w * h).map(|&b| b as f32 / 255.0).collect(), + } + } + + /// Convert colour channels (in the document's mode) to RGB. + fn to_rgb(&self, ch: &[Option>], i: usize) -> [f32; 3] { + let g = |k: usize| ch.get(k).and_then(|c| c.as_ref()).map_or(0.0, |c| c[i]); + let rgb = match self.mode { + ColorMode::Rgb => [g(0), g(1), g(2)], + ColorMode::Cmyk => { + // Stored inverted (1 = no ink). + let k = g(3); + [g(0) * k, g(1) * k, g(2) * k] + } + ColorMode::Indexed if self.palette.len() >= 768 => { + let idx = (g(0) * 255.0).round() as usize & 255; + [self.palette[idx] as f32 / 255.0, self.palette[256 + idx] as f32 / 255.0, self.palette[512 + idx] as f32 / 255.0] + } + ColorMode::Lab => lab_to_rgb(g(0) * 100.0, g(1) * 255.0 - 128.0, g(2) * 255.0 - 128.0), + _ => [g(0); 3], + }; + if self.depth == 32 && self.mode != ColorMode::Lab { rgb.map(srgb_encode) } else { rgb } + } + + /// The merged (flattened) image. + pub fn composite(&self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let mut r = Reader::at(&self.data, self.merged_offset); + let comp = r.u16()?; + let nch = self.channels as usize; + let cc = self.mode.color_channels(); + let want = (cc + usize::from(self.merged_alpha)).min(nch); + let rowb = self.row_bytes(w); + let mut planes: Vec>> = vec![None; nch]; + match comp { + 0 => { + for p in planes.iter_mut().take(want) { + *p = Some(self.samples(r.bytes(rowb * h)?, w, h)); + } + } + 1 => { + let mut counts = Vec::with_capacity(nch * h); + for _ in 0..nch * h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut out = Vec::with_capacity(rowb * h); + for row in 0..h { + unpack_bits(r.bytes(counts[c * h + row])?, rowb, &mut out)?; + } + *p = Some(self.samples(&out, w, h)); + } + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut plane = v.get(c * rowb * h..(c + 1) * rowb * h).ok_or(Error::Truncated)?.to_vec(); + if comp == 3 { + self.unpredict(&mut plane, w, h); + } + *p = Some(self.samples(&plane, w, h)); + } + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + } + let alpha = if self.merged_alpha && nch > cc { planes[cc].take() } else { None }; + let mut data = Vec::with_capacity(w * h); + for i in 0..w * h { + let [r, g, b] = self.to_rgb(&planes, i); + let a = alpha.as_ref().map_or(1.0, |a| a[i]); + data.push([r, g, b, a]); + } + Ok(Pixels { width: w as u32, height: h as u32, data }) + } + + /// One layer's pixels with its layer mask (and vector-less fill colour) applied to alpha. + /// `canvas`: document-sized with the layer in place; otherwise the layer's own bounds. + pub fn layer_pixels(&self, index: usize, canvas: bool) -> Result { + let l = self.layers.get(index).ok_or_else(|| Error::Invalid(format!("no layer {index}")))?; + let doc = Rect { top: 0, left: 0, bottom: self.height as i32, right: self.width as i32 }; + // Fill layers often have no pixel data: they cover the canvas (or the mask). + let fill_rect = + if l.fill.is_some() && !l.has_pixels() { l.mask.filter(|m| !m.rect.is_empty() && m.default_color == 0).map_or(doc, |m| m.rect) } else { l.rect }; + let out_rect = if canvas { + doc + } else if fill_rect.is_empty() { + Rect { top: 0, left: 0, bottom: 1, right: 1 } + } else { + fill_rect + }; + let (ow, oh) = (out_rect.width() as usize, out_rect.height() as usize); + let mut out = vec![[0.0f32; 4]; ow * oh]; + let src = fill_rect; + let (sw, sh) = (src.width() as usize, src.height() as usize); + if sw > 0 && sh > 0 { + let mut planes: Vec>> = vec![None; self.mode.color_channels()]; + let mut alpha = None; + if l.has_pixels() { + for c in &l.channels { + match c.id { + -1 => alpha = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + id if id >= 0 && (id as usize) < planes.len() => planes[id as usize] = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + _ => {} + } + } + } + let fill = l.fill.map(|c| c.map(|v| v as f32)); + for y in 0..sh { + let dy = src.top + y as i32 - out_rect.top; + if dy < 0 || dy >= oh as i32 { + continue; + } + for x in 0..sw { + let dx = src.left + x as i32 - out_rect.left; + if dx < 0 || dx >= ow as i32 { + continue; + } + let i = y * sw + x; + let (rgb, a) = match fill { + Some(f) if !l.has_pixels() => (f, 1.0), + _ => (self.to_rgb(&planes, i), alpha.as_ref().map_or(1.0, |a| a[i])), + }; + out[dy as usize * ow + dx as usize] = [rgb[0], rgb[1], rgb[2], a]; + } + } + } + // Layer mask: multiplies alpha (also outside the mask rect, by its default colour). + if let Some(m) = l.mask.filter(|m| !m.disabled) { + let mc = l.channels.iter().find(|c| c.id == -2); + let (mw, mh) = (m.rect.width() as usize, m.rect.height() as usize); + let mask = match mc { + Some(c) if mw > 0 && mh > 0 => Some(self.decode_channel(c.offset, c.len, mw, mh)?), + _ => None, + }; + let def = m.default_color as f32 / 255.0; + for y in 0..oh { + let py = out_rect.top + y as i32; + for x in 0..ow { + let px = out_rect.left + x as i32; + let inside = py >= m.rect.top && py < m.rect.bottom && px >= m.rect.left && px < m.rect.right; + let mut v = match (&mask, inside) { + (Some(mk), true) => mk[(py - m.rect.top) as usize * mw + (px - m.rect.left) as usize], + _ => def, + }; + if m.inverted { + v = 1.0 - v; + } + out[y * ow + x][3] *= v; + } + } + } + Ok(Pixels { width: ow as u32, height: oh as u32, data: out }) + } +} + +/// The next additional-information block at `*pos` (signature `8BIM`/`8B64`, key, length): +/// returns (key, data start, data end). Skips up to 3 padding bytes before a signature. +fn next_block(data: &[u8], pos: &mut usize, end: usize, psb: bool) -> Option<([u8; 4], usize, usize)> { + let mut p = *pos; + for _ in 0..4 { + if p + 12 > end { + return None; + } + if &data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64" { + break; + } + p += 1; + } + if p + 12 > end || !(&data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64") { + return None; + } + let mut key = [0u8; 4]; + key.copy_from_slice(&data[p + 4..p + 8]); + let wide = + psb && matches!(&key, b"LMsk" | b"Lr16" | b"Lr32" | b"Layr" | b"Mt16" | b"Mt32" | b"Mtrn" | b"Alph" | b"FMsk" | b"lnk2" | b"FEid" | b"FXid" | b"PxSD"); + let mut r = Reader::at(&data[..end], p + 8); + let len = r.length(wide).ok()?; + let start = r.pos; + let stop = start.checked_add(len)?; + if stop > end { + return None; + } + *pos = stop; + Some((key, start, stop)) +} + +/// PackBits decoding of one row (appends exactly `row` bytes, zero-padded). +fn unpack_bits(src: &[u8], row: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let mut i = 0; + while i < src.len() && out.len() - start < row { + let n = src[i] as i8; + i += 1; + if n >= 0 { + let k = n as usize + 1; + let s = src.get(i..i + k).ok_or(Error::Truncated)?; + out.extend_from_slice(s); + i += k; + } else if n != -128 { + let k = (1 - n as isize) as usize; + let v = *src.get(i).ok_or(Error::Truncated)?; + out.extend(std::iter::repeat_n(v, k)); + i += 1; + } + } + out.resize(start + row, 0); + Ok(()) +} + +/// PackBits encoding of one row. +pub(crate) fn pack_bits(src: &[u8]) -> Vec { + let mut out = vec![]; + let mut i = 0; + while i < src.len() { + // run of identical bytes + let mut run = 1; + while i + run < src.len() && src[i + run] == src[i] && run < 128 { + run += 1; + } + if run >= 2 { + out.push((1 - run as isize) as i8 as u8); + out.push(src[i]); + i += run; + continue; + } + // literal run + let start = i; + let mut n = 0; + while i < src.len() && n < 128 { + if i + 1 < src.len() && src[i + 1] == src[i] { + break; + } + i += 1; + n += 1; + } + if n == 0 { + i += 1; + n = 1; + } + out.push((n - 1) as u8); + out.extend_from_slice(&src[start..start + n]); + } + out +} + +fn srgb_encode(v: f32) -> f32 { + if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } +} + +/// CIE L*a*b* (D50) to sRGB (D65 via the Bradford-adapted matrix), clamped. +fn lab_to_rgb(l: f32, a: f32, b: f32) -> [f32; 3] { + let fy = (l + 16.0) / 116.0; + let fx = fy + a / 500.0; + let fz = fy - b / 200.0; + let f = |t: f32| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f32 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + let (x, y, z) = (0.964_22 * f(fx), f(fy), 0.825_21 * f(fz)); + // XYZ (D50) → linear sRGB. + let r = 3.133_856 * x - 1.616_867 * y - 0.490_615 * z; + let g = -0.978_768 * x + 1.916_142 * y + 0.033_454 * z; + let bb = 0.071_945 * x - 0.228_991 * y + 1.405_243 * z; + [r, g, bb].map(|v| srgb_encode(v.clamp(0.0, 1.0))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/psd/src/read.rs b/crates/psd/src/read.rs new file mode 100644 index 000000000..97651b2ba --- /dev/null +++ b/crates/psd/src/read.rs @@ -0,0 +1,86 @@ +//! Big-endian byte reader. + +use crate::{Error, Result}; + +#[derive(Clone)] +pub struct Reader<'a> { + pub data: &'a [u8], + pub pos: usize, +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data, pos: 0 } + } + pub fn at(data: &'a [u8], pos: usize) -> Self { + Reader { data, pos } + } + pub fn remaining(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> { + if self.remaining() < n { + return Err(Error::Truncated); + } + let s = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + pub fn skip(&mut self, n: usize) -> Result<()> { + self.bytes(n).map(|_| ()) + } + pub fn u8(&mut self) -> Result { + Ok(self.bytes(1)?[0]) + } + pub fn u16(&mut self) -> Result { + let b = self.bytes(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + pub fn i16(&mut self) -> Result { + Ok(self.u16()? as i16) + } + pub fn u32(&mut self) -> Result { + let b = self.bytes(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + pub fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + pub fn u64(&mut self) -> Result { + let b = self.bytes(8)?; + let mut a = [0u8; 8]; + a.copy_from_slice(b); + Ok(u64::from_be_bytes(a)) + } + pub fn f32(&mut self) -> Result { + Ok(f32::from_bits(self.u32()?)) + } + pub fn f64(&mut self) -> Result { + Ok(f64::from_bits(self.u64()?)) + } + pub fn tag(&mut self) -> Result<[u8; 4]> { + let b = self.bytes(4)?; + Ok([b[0], b[1], b[2], b[3]]) + } + /// A length field: 4 bytes in PSD, 8 bytes in PSB (`wide`). + pub fn length(&mut self, wide: bool) -> Result { + let v = if wide { self.u64()? } else { self.u32()? as u64 }; + usize::try_from(v).map_err(|_| Error::Invalid("length".into())) + } + /// A Pascal string padded so that length byte + text is a multiple of `pad`. + pub fn pascal(&mut self, pad: usize) -> Result { + let n = self.u8()? as usize; + let s = self.bytes(n)?; + let total = 1 + n; + let padded = total.div_ceil(pad) * pad; + self.skip(padded - total)?; + // Pascal strings are in the system (Mac Roman) encoding; ASCII covers the common case. + Ok(s.iter().map(|&b| if b < 0x80 { b as char } else { mac_roman(b) }).collect()) + } +} + +/// Mac OS Roman upper half (0x80..=0xFF), as published by the Unicode Consortium mapping table. +fn mac_roman(b: u8) -> char { + const T: &str = "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ"; + T.chars().nth((b - 0x80) as usize).unwrap_or('?') +} diff --git a/crates/psd/src/tests.rs b/crates/psd/src/tests.rs new file mode 100644 index 000000000..68680fe71 --- /dev/null +++ b/crates/psd/src/tests.rs @@ -0,0 +1,189 @@ +use super::write::*; +use super::*; + +fn gradient(w: u32, h: u32) -> Vec<[f32; 4]> { + (0..w * h).map(|i| [(i % w) as f32 / (w - 1).max(1) as f32, (i / w) as f32 / (h - 1).max(1) as f32, 0.25, 1.0]).collect() +} + +fn close(a: &[[f32; 4]], b: &[[f32; 4]], tol: f32) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (0..4).all(|c| (x[c] - y[c]).abs() <= tol)) +} + +fn doc_with_layers(depth: u16, mode: ColorMode, rle: bool) -> WDoc { + let mut d = WDoc::new(40, 30); + d.depth = depth; + d.mode = mode; + d.rle = rle; + let mut bg = WLayer::pixels("Background", Rect::new(0, 0, 40, 30), gradient(40, 30)); + bg.opacity = 255; + let mut red = WLayer::solid("Red box", Rect::new(5, 6, 10, 8), [1.0, 0.0, 0.0, 1.0]).with_blend(b"mul ").with_opacity(128); + red.hidden = true; + let mut masked = WLayer::solid("Masked", Rect::new(20, 10, 12, 12), [0.0, 0.5, 1.0, 1.0]).with_blend(b"scrn"); + // Mask: left half opaque, right half transparent; default (outside) black. + let mrect = Rect::new(20, 10, 12, 12); + masked.mask = Some(WMask { rect: mrect, data: (0..144).map(|i| if i % 12 < 6 { 1.0 } else { 0.0 }).collect(), default_color: 0, disabled: false }); + d.layers = vec![bg, red, masked]; + d +} + +#[test] +fn layers_round_trip_8bit_rle() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + assert!(is_psd(&bytes)); + let p = Psd::parse(bytes).unwrap(); + assert_eq!((p.width, p.height, p.depth, p.mode), (40, 30, 8, ColorMode::Rgb)); + assert_eq!(p.layers.len(), 3); + let names: Vec<&str> = p.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Background", "Red box", "Masked"]); + assert_eq!(p.layers[1].blend_key(), "mul "); + assert_eq!(p.layers[1].opacity, 128); + assert!(p.layers[1].hidden && !p.layers[0].hidden); + assert_eq!(p.layers[2].blend_key(), "scrn"); + assert_eq!(p.layers[1].rect, Rect::new(5, 6, 10, 8)); + // Pixels (8-bit quantisation). + let bg = p.layer_pixels(0, false).unwrap(); + assert!(close(&bg.data, &gradient(40, 30), 1.0 / 255.0)); + // Retain layer size vs canvas. + let red = p.layer_pixels(1, false).unwrap(); + assert_eq!((red.width, red.height), (10, 8)); + let red_c = p.layer_pixels(1, true).unwrap(); + assert_eq!((red_c.width, red_c.height), (40, 30)); + assert_eq!(red_c.data[6 * 40 + 5], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(red_c.data[0][3], 0.0); + // Layer mask becomes alpha. + let m = p.layer_pixels(2, false).unwrap(); + assert_eq!(m.data[0][3], 1.0); + assert_eq!(m.data[11][3], 0.0); + // Merged composite. + let c = p.composite().unwrap(); + assert_eq!((c.width, c.height), (40, 30)); + assert!(p.merged_alpha); +} + +#[test] +fn layers_round_trip_16bit_raw_and_cmyk_gray() { + for (depth, mode, rle) in + [(16, ColorMode::Rgb, false), (16, ColorMode::Rgb, true), (8, ColorMode::Cmyk, true), (16, ColorMode::Cmyk, false), (8, ColorMode::Grayscale, false)] + { + let bytes = write(&doc_with_layers(depth, mode, rle)); + let p = Psd::parse(bytes).unwrap_or_else(|e| panic!("{depth} {mode:?} {rle}: {e}")); + assert_eq!(p.layers.len(), 3, "{depth} {mode:?}"); + assert_eq!(p.depth, depth); + let bg = p.layer_pixels(0, true).unwrap(); + let tol = if depth == 16 { 1.0 / 30000.0 } else { 1.5 / 255.0 }; + let want: Vec<[f32; 4]> = + if mode == ColorMode::Grayscale { gradient(40, 30).iter().map(|p| [p[0], p[0], p[0], 1.0]).collect() } else { gradient(40, 30) }; + assert!(close(&bg.data, &want, tol), "{depth} {mode:?} {rle}: {:?} vs {:?}", bg.data[41], want[41]); + let c = p.composite().unwrap(); + assert_eq!(c.data.len(), 1200); + } +} + +#[test] +fn groups_build_a_tree() { + let mut d = WDoc::new(16, 16); + d.layers = vec![ + WLayer::solid("Bottom", Rect::new(0, 0, 16, 16), [1.0; 4]), + WLayer::group_end(), + WLayer::solid("Inner A", Rect::new(0, 0, 4, 4), [1.0, 0.0, 0.0, 1.0]), + WLayer::group_end(), + WLayer::solid("Deep", Rect::new(2, 2, 4, 4), [0.0, 1.0, 0.0, 1.0]), + WLayer::group("Nested", false, *b"norm"), + WLayer::group("Group 1", true, *b"pass"), + WLayer::solid("Top", Rect::new(8, 8, 4, 4), [0.0, 0.0, 1.0, 1.0]), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[6].section, Section::OpenFolder); + assert_eq!(p.layers[6].blend_key(), "pass"); + assert_eq!(p.layers[5].section, Section::ClosedFolder); + let t = p.tree(); + assert_eq!( + t, + vec![ + Node::Layer(7), + Node::Group { layer: 6, children: vec![Node::Group { layer: 5, children: vec![Node::Layer(4)] }, Node::Layer(2)] }, + Node::Layer(0) + ] + ); +} + +#[test] +fn text_effects_fill_and_adjustments() { + let mut d = WDoc::new(64, 32); + let fx = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 1.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 7.0)), + ), + ); + d.layers = vec![ + WLayer::empty("Fill", vec![solid_color_block([0.0, 1.0, 0.0])]), + WLayer::empty("Title", vec![text_block("Hello\nWorld", [10.0, 20.0], "Inter-Bold", 18.0, [1.0, 0.0, 0.0], 2)]).with_block(effects_block(&fx)), + WLayer::empty("Hue", vec![hue_sat_block(30, -20, 10)]), + WLayer::empty("Lev", vec![levels_block(10, 240, 5, 250, 1.2)]), + WLayer::empty("Inv", vec![invert_block()]), + WLayer::empty("BC", vec![brightness_block(20, -10)]), + WLayer::empty("Exp", vec![exposure_block(1.5, 0.0, 1.0)]), + WLayer::solid("Vec", Rect::new(0, 0, 64, 32), [1.0; 4]).with_block(vector_mask_block( + &[vec![[[8.0, 4.0]; 3], [[40.0, 4.0]; 3], [[40.0, 20.0]; 3]]], + 64, + 32, + false, + )), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[0].fill, Some([0.0, 1.0, 0.0])); + let fill = p.layer_pixels(0, false).unwrap(); + assert_eq!((fill.width, fill.height), (64, 32)); + assert_eq!(fill.data[5], [0.0, 1.0, 0.0, 1.0]); + let t = p.layers[1].text.as_ref().unwrap(); + assert_eq!(t.text, "Hello\nWorld"); + assert_eq!(t.transform[4..], [10.0, 20.0]); + assert_eq!(t.style.font.as_deref(), Some("Inter-Bold")); + assert_eq!(t.style.size, Some(18.0)); + assert_eq!(t.style.color, Some([1.0, 0.0, 0.0])); + assert_eq!(t.style.justification, Some(2)); + let e = p.layers[1].effects.as_ref().unwrap(); + let ds = e.obj("DrSh").unwrap(); + assert_eq!(ds.enum_value("Md "), Some("Mltp")); + assert_eq!(ds.num("Dstn"), Some(7.0)); + assert_eq!(p.layers[2].adjustment, Some(Adjustment::HueSaturation { hue: 30.0, saturation: -20.0, lightness: 10.0, colorize: false })); + assert_eq!(p.layers[3].adjustment, Some(Adjustment::Levels { in_black: 10.0, in_white: 240.0, gamma: 1.2, out_black: 5.0, out_white: 250.0 })); + assert_eq!(p.layers[4].adjustment, Some(Adjustment::Invert)); + assert_eq!(p.layers[5].adjustment, Some(Adjustment::BrightnessContrast { brightness: 20.0, contrast: -10.0, legacy: true })); + assert_eq!(p.layers[6].adjustment, Some(Adjustment::Exposure { exposure: 1.5, offset: 0.0, gamma: 1.0 })); + let vm = p.layers[7].vector_mask.as_ref().unwrap(); + assert_eq!(vm.subpaths.len(), 1); + assert!(vm.subpaths[0].closed); + let k = &vm.subpaths[0].knots; + assert_eq!(k.len(), 3); + assert!((k[1][1][0] - 40.0).abs() < 1e-4 && (k[2][1][1] - 20.0).abs() < 1e-4); +} + +#[test] +fn packbits_round_trip() { + for src in [vec![], vec![1u8], vec![5; 300], (0..=255).collect::>(), vec![1, 1, 2, 3, 3, 3, 4, 5, 5]] { + let enc = pack_bits(&src); + let mut out = vec![]; + unpack_bits(&enc, src.len(), &mut out).unwrap(); + assert_eq!(out, src); + } +} + +#[test] +fn truncated_and_garbage_input_errors() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + for n in [0, 3, 10, 26, 40, 100, bytes.len() / 2] { + let r = Psd::parse(bytes[..n].to_vec()); + if let Ok(p) = r { + let _ = p.composite(); + } + } + assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err()); + assert!(Psd::parse(b"PNG".to_vec()).is_err()); +} diff --git a/crates/psd/src/write.rs b/crates/psd/src/write.rs new file mode 100644 index 000000000..fb78cc78e --- /dev/null +++ b/crates/psd/src/write.rs @@ -0,0 +1,491 @@ +//! A minimal Photoshop document writer, per the same specification as the reader: RGB, CMYK or +//! Grayscale, 8 or 16 bits, raw or RLE channels, pixel layers with layer masks, groups and any +//! additional-information blocks (helpers below build text, effects, fill and adjustment +//! blocks). Used to generate test fixtures; not a full exporter. + +use crate::descriptor::{self, DValue, Descriptor}; +use crate::{ColorMode, Rect, Section, engine_data, pack_bits}; + +/// A layer mask for [`WLayer`]. +#[derive(Clone, Debug)] +pub struct WMask { + pub rect: Rect, + /// `rect.width() * rect.height()` values in 0..1. + pub data: Vec, + pub default_color: u8, + pub disabled: bool, +} + +/// A layer to write. +#[derive(Clone, Debug)] +pub struct WLayer { + pub name: String, + pub rect: Rect, + /// Straight RGBA, `rect.width() * rect.height()` pixels. + pub pixels: Vec<[f32; 4]>, + pub blend: [u8; 4], + pub opacity: u8, + pub hidden: bool, + pub clipping: bool, + pub mask: Option, + pub section: Section, + /// Extra additional-information blocks (key, data). + pub blocks: Vec<([u8; 4], Vec)>, +} + +impl WLayer { + /// A pixel layer. + pub fn pixels(name: &str, rect: Rect, pixels: Vec<[f32; 4]>) -> WLayer { + WLayer { + name: name.into(), + rect, + pixels, + blend: *b"norm", + opacity: 255, + hidden: false, + clipping: false, + mask: None, + section: Section::Layer, + blocks: vec![], + } + } + /// A layer filled with one colour. + pub fn solid(name: &str, rect: Rect, rgba: [f32; 4]) -> WLayer { + let n = (rect.width() * rect.height()) as usize; + WLayer::pixels(name, rect, vec![rgba; n]) + } + /// The top record of a group (its children come *before* it in file order, after a + /// [`WLayer::group_end`]). + pub fn group(name: &str, open: bool, blend: [u8; 4]) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.section = if open { Section::OpenFolder } else { Section::ClosedFolder }; + l.blend = blend; + l + } + /// The hidden record that closes a group (written below its children). + pub fn group_end() -> WLayer { + let mut l = WLayer::pixels("", Rect::default(), vec![]); + l.section = Section::Divider; + l + } + /// A layer without pixels carrying `blocks` (adjustment and fill layers, type layers). + pub fn empty(name: &str, blocks: Vec<([u8; 4], Vec)>) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.blocks = blocks; + l + } + pub fn with_blend(mut self, key: &[u8; 4]) -> WLayer { + self.blend = *key; + self + } + pub fn with_opacity(mut self, o: u8) -> WLayer { + self.opacity = o; + self + } + pub fn with_block(mut self, b: ([u8; 4], Vec)) -> WLayer { + self.blocks.push(b); + self + } +} + +/// A document to write. +#[derive(Clone, Debug)] +pub struct WDoc { + pub width: u32, + pub height: u32, + /// 8 or 16. + pub depth: u16, + /// Rgb, Cmyk or Grayscale. + pub mode: ColorMode, + pub rle: bool, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers + /// with Normal blending. + pub composite: Option>, +} + +impl WDoc { + pub fn new(width: u32, height: u32) -> WDoc { + WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None } + } +} + +fn u16b(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} +fn u32b(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Colour channels for a straight RGB pixel in the document mode. +fn mode_channels(mode: ColorMode, p: [f32; 4]) -> Vec { + match mode { + ColorMode::Cmyk => { + // Stored values are inverted ink amounts: k = max, c = r / max (reader: r = c * k). + let m = p[0].max(p[1]).max(p[2]); + if m <= 0.0 { vec![0.0, 0.0, 0.0, 0.0] } else { vec![p[0] / m, p[1] / m, p[2] / m, m] } + } + ColorMode::Grayscale => vec![p[0]], + _ => vec![p[0], p[1], p[2]], + } +} + +fn encode_samples(v: &[f32], depth: u16) -> Vec { + let mut out = Vec::with_capacity(v.len() * if depth == 16 { 2 } else { 1 }); + for &x in v { + let x = x.clamp(0.0, 1.0); + if depth == 16 { + u16b(&mut out, (x * 65535.0).round() as u16); + } else { + out.push((x * 255.0).round() as u8); + } + } + out +} + +/// One channel's data (compression field + data) for a `w`×`h` plane. +fn channel_data(v: &[f32], w: usize, h: usize, depth: u16, rle: bool) -> Vec { + let raw = encode_samples(v, depth); + let mut out = vec![]; + let rowb = raw.len() / h.max(1); + if !rle || w == 0 || h == 0 { + u16b(&mut out, 0); + out.extend_from_slice(&raw); + return out; + } + u16b(&mut out, 1); + let rows: Vec> = raw.chunks(rowb.max(1)).map(pack_bits).collect(); + for r in &rows { + u16b(&mut out, r.len() as u16); + } + for r in rows { + out.extend_from_slice(&r); + } + out +} + +fn pascal4(out: &mut Vec, s: &str) { + let b: Vec = s.bytes().map(|c| if c < 0x80 { c } else { b'?' }).take(255).collect(); + out.push(b.len() as u8); + out.extend_from_slice(&b); + let total = 1 + b.len(); + out.resize(out.len() + total.div_ceil(4) * 4 - total, 0); +} + +fn block(out: &mut Vec, key: &[u8; 4], data: &[u8]) { + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(key); + let padded = data.len().div_ceil(2) * 2; + u32b(out, padded as u32); + out.extend_from_slice(data); + out.resize(out.len() + padded - data.len(), 0); +} + +fn layer_info(doc: &WDoc) -> Vec { + let mut out = vec![]; + out.extend_from_slice(&(-(doc.layers.len() as i16)).to_be_bytes()); + let mut chan_data: Vec)>> = vec![]; + for l in &doc.layers { + let (w, h) = (l.rect.width() as usize, l.rect.height() as usize); + let mut chans: Vec<(i16, Vec)> = vec![]; + let n = w * h; + let alpha: Vec = l.pixels.iter().take(n).map(|p| p[3]).collect(); + chans.push((-1, channel_data(&alpha, w, h, doc.depth, doc.rle))); + let cc = doc.mode.color_channels(); + let planes: Vec> = l.pixels.iter().take(n).map(|p| mode_channels(doc.mode, *p)).collect(); + for c in 0..cc { + let plane: Vec = planes.iter().map(|v| v[c]).collect(); + chans.push((c as i16, channel_data(&plane, w, h, doc.depth, doc.rle))); + } + if let Some(m) = &l.mask { + chans.push((-2, channel_data(&m.data, m.rect.width() as usize, m.rect.height() as usize, doc.depth, doc.rle))); + } + // record + for v in [l.rect.top, l.rect.left, l.rect.bottom, l.rect.right] { + out.extend_from_slice(&v.to_be_bytes()); + } + u16b(&mut out, chans.len() as u16); + for (id, d) in &chans { + out.extend_from_slice(&id.to_be_bytes()); + u32b(&mut out, d.len() as u32); + } + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(&l.blend); + out.push(l.opacity); + out.push(l.clipping as u8); + let mut flags = 8u8; + if l.hidden { + flags |= 2; + } + out.push(flags); + out.push(0); + let mut extra = vec![]; + match &l.mask { + Some(m) => { + u32b(&mut extra, 20); + for v in [m.rect.top, m.rect.left, m.rect.bottom, m.rect.right] { + extra.extend_from_slice(&v.to_be_bytes()); + } + extra.push(m.default_color); + extra.push(if m.disabled { 2 } else { 0 }); + extra.extend_from_slice(&[0, 0]); + } + None => u32b(&mut extra, 0), + } + u32b(&mut extra, 0); // blending ranges + pascal4(&mut extra, &l.name); + let luni = unicode_plain(&l.name); + block(&mut extra, b"luni", &luni); + match l.section { + Section::Layer => {} + s => { + let mut d = vec![]; + u32b( + &mut d, + match s { + Section::OpenFolder => 1, + Section::ClosedFolder => 2, + _ => 3, + }, + ); + d.extend_from_slice(b"8BIM"); + d.extend_from_slice(&l.blend); + block(&mut extra, b"lsct", &d); + } + } + for (k, d) in &l.blocks { + block(&mut extra, k, d); + } + u32b(&mut out, extra.len() as u32); + out.extend_from_slice(&extra); + chan_data.push(chans); + } + for chans in chan_data { + for (_, d) in chans { + out.extend_from_slice(&d); + } + } + if out.len() % 2 == 1 { + out.push(0); + } + out +} + +fn unicode_plain(s: &str) -> Vec { + let v: Vec = s.encode_utf16().collect(); + let mut out = vec![]; + u32b(&mut out, v.len() as u32); + for c in v { + u16b(&mut out, c); + } + out +} + +/// Normal "over" of the visible pixel layers (groups ignored), straight alpha. +fn flatten(doc: &WDoc) -> Vec<[f32; 4]> { + let (w, h) = (doc.width as i32, doc.height as i32); + let mut acc = vec![[0.0f32; 4]; (w * h) as usize]; + for l in doc.layers.iter().filter(|l| !l.hidden && l.section == Section::Layer) { + let lw = l.rect.width() as i32; + for (i, p) in l.pixels.iter().enumerate() { + let (x, y) = (l.rect.left + i as i32 % lw.max(1), l.rect.top + i as i32 / lw.max(1)); + if x < 0 || y < 0 || x >= w || y >= h { + continue; + } + let a = p[3] * l.opacity as f32 / 255.0; + let d = &mut acc[(y * w + x) as usize]; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (p[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +/// Serialise a document. +pub fn write(doc: &WDoc) -> Vec { + let cc = doc.mode.color_channels(); + let mut out = vec![]; + out.extend_from_slice(b"8BPS"); + u16b(&mut out, 1); + out.extend_from_slice(&[0; 6]); + u16b(&mut out, (cc + 1) as u16); + u32b(&mut out, doc.height); + u32b(&mut out, doc.width); + u16b(&mut out, doc.depth); + u16b(&mut out, doc.mode.to_u16()); + u32b(&mut out, 0); // colour mode data + u32b(&mut out, 0); // image resources + // Layer and mask information. + let mut lm = vec![]; + if !doc.layers.is_empty() { + let li = layer_info(doc); + if doc.depth == 16 { + u32b(&mut lm, 0); + u32b(&mut lm, 0); // global layer mask info + let mut padded = li; + while !padded.len().is_multiple_of(4) { + padded.push(0); + } + lm.extend_from_slice(b"8BIM"); + lm.extend_from_slice(b"Lr16"); + u32b(&mut lm, padded.len() as u32); + lm.extend_from_slice(&padded); + } else { + u32b(&mut lm, li.len() as u32); + lm.extend_from_slice(&li); + u32b(&mut lm, 0); + } + } + u32b(&mut out, lm.len() as u32); + out.extend_from_slice(&lm); + // Merged image. + let comp = doc.composite.clone().unwrap_or_else(|| flatten(doc)); + let (w, h) = (doc.width as usize, doc.height as usize); + let planes: Vec> = comp.iter().map(|p| mode_channels(doc.mode, *p)).collect(); + let mut chans: Vec> = (0..cc).map(|c| planes.iter().map(|v| v[c]).collect()).collect(); + chans.push(comp.iter().map(|p| p[3]).collect()); + if doc.rle { + u16b(&mut out, 1); + let rows: Vec>> = + chans.iter().map(|c| encode_samples(c, doc.depth).chunks((w * if doc.depth == 16 { 2 } else { 1 }).max(1)).map(pack_bits).collect()).collect(); + for c in &rows { + for r in c { + u16b(&mut out, r.len() as u16); + } + } + for c in rows { + for r in c { + out.extend_from_slice(&r); + } + } + } else { + u16b(&mut out, 0); + for c in &chans { + out.extend_from_slice(&encode_samples(c, doc.depth)); + } + } + let _ = h; + out +} + +// ---------------------------------------------------------------- block helpers + +/// `TySh`: a type layer with one style run; the text origin sits at `origin`. +pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [1.0, 0.0, 0.0, 1.0, origin[0], origin[1]] { + d.extend_from_slice(&f64::to_be_bytes(v)); + } + u16b(&mut d, 50); + u32b(&mut d, 16); + let desc = Descriptor::new("TxLr") + .with("Txt ", DValue::Text(text.replace('\n', "\r"))) + .with("textGridding", DValue::Enum("textGridding".into(), "None".into())) + .with("Ornt", DValue::Enum("Ornt".into(), "Hrzn".into())) + .with("EngineData", DValue::Raw(engine_data::minimal(text, font, size, color, justification))); + descriptor::write_descriptor(&mut d, &desc); + // Warp: version, descriptor version, an empty warp descriptor. + u16b(&mut d, 1); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("warp").with("warpStyle", DValue::Enum("warpStyle".into(), "warpNone".into()))); + for _ in 0..4 { + u32b(&mut d, 0); + } + (*b"TySh", d) +} + +/// `lfx2`: layer effects from an effects descriptor (`null` class with `DrSh`, `FrFX`…). +pub fn effects_block(desc: &Descriptor) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 0); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, desc); + (*b"lfx2", d) +} + +/// `SoCo`: solid colour fill. +pub fn solid_color_block(rgb: [f64; 3]) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("null").with("Clr ", DValue::Descriptor(Descriptor::rgb(rgb)))); + (*b"SoCo", d) +} + +/// `hue2`: Hue/Saturation (master). +pub fn hue_sat_block(hue: i16, sat: i16, light: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + d.extend_from_slice(&[0, 0]); + for v in [0i16, 0, 0, hue, sat, light] { + d.extend_from_slice(&v.to_be_bytes()); + } + // Six colour ranges (unused). + d.resize(d.len() + 6 * 24, 0); + (*b"hue2", d) +} + +/// `levl`: Levels (composite record). +pub fn levels_block(in_black: i16, in_white: i16, out_black: i16, out_white: i16, gamma: f64) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + for v in [in_black, in_white, out_black, out_white, (gamma * 100.0).round() as i16] { + d.extend_from_slice(&v.to_be_bytes()); + } + for _ in 0..28 { + for v in [0i16, 255, 0, 255, 100] { + d.extend_from_slice(&v.to_be_bytes()); + } + } + (*b"levl", d) +} + +pub fn invert_block() -> ([u8; 4], Vec) { + (*b"nvrt", vec![]) +} + +pub fn brightness_block(brightness: i16, contrast: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + for v in [brightness, contrast, 127] { + d.extend_from_slice(&v.to_be_bytes()); + } + d.push(0); + (*b"brit", d) +} + +pub fn exposure_block(exposure: f32, offset: f32, gamma: f32) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [exposure, offset, gamma] { + d.extend_from_slice(&v.to_be_bytes()); + } + (*b"expA", d) +} + +/// `vmsk`: a vector mask from closed polygons/curves given as (in, anchor, out) knots in pixels. +pub fn vector_mask_block(subpaths: &[Vec<[[f64; 2]; 3]>], width: u32, height: u32, inverted: bool) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 3); + u32b(&mut d, inverted as u32); + // Path fill rule record. + u16b(&mut d, 6); + d.resize(d.len() + 24, 0); + let fx = |v: f64| ((v * (1 << 24) as f64).round() as i32).to_be_bytes(); + for sp in subpaths { + u16b(&mut d, 0); + u16b(&mut d, sp.len() as u16); + d.resize(d.len() + 22, 0); + for k in sp { + u16b(&mut d, 1); + for p in k { + d.extend_from_slice(&fx(p[1] / height as f64)); + d.extend_from_slice(&fx(p[0] / width as f64)); + } + } + } + (*b"vmsk", d) +} diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 4c8a9787a..cdaa85c16 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +lz4_flex = { version = "0.11", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] } kurbo = { workspace = true } rayon = { workspace = true } web-time = { workspace = true } diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 4884411f6..4b872ad24 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -330,6 +330,7 @@ mod tests { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f)); let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8cd27d598..23702fd69 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -25,9 +25,13 @@ use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup}; use crate::eval::EvalCtx; -/// Thread-safe, memory-budgeted cache of processed layer buffers. +/// Thread-safe, memory-budgeted cache of processed layer buffers, optionally backed by the +/// persistent [`DiskCache`](crate::disk_cache::DiskCache): a memory miss reads the disk, and +/// buffers that were slow to render are written there (keys salted per project, see +/// [`LayerCache::set_disk`]). pub struct LayerCache { inner: Mutex, + disk: Mutex, u64)>>, } struct Entry { @@ -43,6 +47,8 @@ struct Inner { clock: u64, hits: u64, misses: u64, + /// When each recent miss happened (to time the render that follows it). + missed_at: HashMap, } /// Hit/miss counters and memory use (for perf readouts and tests). @@ -63,7 +69,37 @@ impl Default for LayerCache { impl LayerCache { /// A cache holding at most `budget` bytes of pixels. pub fn new(budget: usize) -> LayerCache { - LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0 }) } + LayerCache { + inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }), + disk: Mutex::new(None), + } + } + + /// Attach (or detach) the disk cache. `salt` separates projects and footage versions (see + /// `footage_salt`); keys on disk are `(salt, key)`. + pub fn set_disk(&self, disk: Option<(Arc, u64)>) { + if let Ok(mut d) = self.disk.lock() { + *d = disk; + } + } + + /// Change the disk salt (after footage changed) keeping the attached disk cache. + pub fn set_disk_salt(&self, salt: u64) { + if let Ok(mut d) = self.disk.lock() + && let Some((_, s)) = d.as_mut() + { + *s = salt; + } + } + + pub fn disk(&self) -> Option> { + self.disk.lock().ok()?.as_ref().map(|d| d.0.clone()) + } + + fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { + let d = self.disk.lock().ok()?; + let (dc, salt) = d.as_ref()?; + Some((dc.clone(), ((*salt as u128) << 64) | key as u128)) } pub fn get(&self, key: u64) -> Option> { @@ -79,12 +115,32 @@ impl LayerCache { } None => { g.misses += 1; - None + if g.missed_at.len() > 4096 { + g.missed_at.clear(); + } + g.missed_at.insert(key, web_time::Instant::now()); + drop(g); + // Disk cache: a hit comes back into memory. + let (dc, dk) = self.disk_key(key)?; + let buf = Arc::new(dc.get_layer(dk)?); + self.insert_mem(key, buf.clone()); + Some(buf) } } } pub fn insert(&self, key: u64, buf: Arc) { + let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key)); + if let Some((dc, dk)) = self.disk_key(key) { + let slow = started.is_some_and(|t| t.elapsed().as_secs_f64() * 1e3 >= dc.min_layer_ms() as f64); + if slow { + dc.put_layer(dk, &buf); + } + } + self.insert_mem(key, buf); + } + + fn insert_mem(&self, key: u64, buf: Arc) { let bytes = buf.img.data.len() * std::mem::size_of::(); let Ok(mut g) = self.inner.lock() else { return }; if bytes > g.budget / 4 { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs new file mode 100644 index 000000000..1afc9bbb2 --- /dev/null +++ b/crates/render/src/disk_cache.rs @@ -0,0 +1,829 @@ +//! The persistent disk cache (Settings ▸ Media & Disk Cache ▸ Disk Cache): RAM-preview frames +//! and layer cache entries kept on disk between sessions, like After Effects' disk cache. +//! +//! * Entries are files keyed by a 128-bit content hash (`/v1//<2 hex>/<32 hex>.ecc`): +//! frames by the hash of the composition's content (the comp and everything it uses, footage +//! files' size and modification time included) plus frame, scale and view; layer buffers by +//! the layer cache's content key salted with the project's footage fingerprint. A changed +//! project therefore never reads stale pixels: its keys simply change. +//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer +//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh +//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! are deleted. +//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the +//! folder when the cache opens (file modification times order it), so the cache survives +//! restarts; reads refresh a file's time. +//! * Safe concurrent access: writes go to a unique temporary file that is renamed into place +//! (atomic on every desktop OS), so readers in this or another process never see a partial +//! entry; a missing or damaged file is just a miss. Writes run on a background thread with a +//! bounded queue (a full queue drops the write rather than stalling rendering). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use effectcraft_effects::Buf; +use effectcraft_raster::Image; + +/// File format version (part of the folder layout). +const VERSION: &str = "v1"; +const MAGIC: &[u8; 4] = b"ECDC"; +/// Pending writes before new ones are dropped. +const QUEUE_LIMIT: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Kind { + Frame, + Layer, +} + +impl Kind { + fn dir(self) -> &'static str { + match self { + Kind::Frame => "frames", + Kind::Layer => "layers", + } + } + fn tag(self) -> u8 { + match self { + Kind::Frame => 1, + Kind::Layer => 2, + } + } +} + +/// A cached viewer frame: 8-bit premultiplied RGBA. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame8 { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} + +/// Counters for tests, the Info panel and `cache.diskStats`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiskStats { + pub entries: usize, + pub bytes: u64, + pub max_bytes: u64, + pub hits: u64, + pub misses: u64, + pub writes: u64, + pub evictions: u64, + pub dropped: u64, +} + +struct Meta { + bytes: u64, + last_use: u64, +} + +#[derive(Default)] +struct Index { + map: HashMap<(Kind, u128), Meta>, + total: u64, + clock: u64, +} + +enum Job { + Write(Kind, u128, Vec), +} + +struct Queue { + jobs: std::collections::VecDeque, + busy: bool, +} + +pub struct DiskCache { + dir: PathBuf, + index: Mutex, + max_bytes: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + writes: AtomicU64, + evictions: AtomicU64, + dropped: AtomicU64, + tmp_counter: AtomicU64, + queue: Arc<(Mutex, Condvar)>, + /// Layer buffers that took less than this to render are not written (ms). + min_layer_ms: AtomicU64, +} + +/// FNV-1a 128-bit (content keys). +#[derive(Clone, Copy)] +pub struct Hash128(u128); + +impl Default for Hash128 { + fn default() -> Self { + Hash128(0x6c62_272e_07bb_0142_62b8_2175_6295_c58d) + } +} + +impl Hash128 { + pub fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 ^= b as u128; + self.0 = self.0.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + } + pub fn write_u64(&mut self, v: u64) { + self.write(&v.to_le_bytes()); + } + pub fn finish(&self) -> u128 { + self.0 + } +} + +fn checksum(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// The platform's default disk cache folder. +pub fn default_folder() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from); + if cfg!(target_os = "macos") + && let Some(h) = &home + { + return h.join("Library/Caches/EffectCraft/Disk Cache"); + } + if cfg!(windows) + && let Some(l) = std::env::var_os("LOCALAPPDATA") + { + return PathBuf::from(l).join("EffectCraft").join("Disk Cache"); + } + if let Some(x) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(x).join("effectcraft/disk-cache"); + } + home.map(|h| h.join(".cache/effectcraft/disk-cache")).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-disk-cache")) +} + +impl DiskCache { + /// Open (or create) the cache in `folder`, holding at most `max_bytes`. + pub fn open(folder: impl AsRef, max_bytes: u64) -> std::io::Result> { + let dir = folder.as_ref().join(VERSION); + std::fs::create_dir_all(&dir)?; + let dc = Arc::new(DiskCache { + dir, + index: Mutex::new(Index::default()), + max_bytes: AtomicU64::new(max_bytes), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + writes: AtomicU64::new(0), + evictions: AtomicU64::new(0), + dropped: AtomicU64::new(0), + tmp_counter: AtomicU64::new(0), + queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: false }), Condvar::new())), + min_layer_ms: AtomicU64::new(30), + }); + dc.rescan(); + dc.evict(); + #[cfg(not(target_arch = "wasm32"))] + { + let weak = Arc::downgrade(&dc); + let q = dc.queue.clone(); + std::thread::Builder::new().name("ec-disk-cache".into()).spawn(move || writer(weak, q))?; + } + Ok(dc) + } + + pub fn folder(&self) -> &Path { + self.dir.parent().unwrap_or(&self.dir) + } + + /// Rebuild the index from the folder (oldest files first). + pub fn rescan(&self) { + let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; + for kind in [Kind::Frame, Kind::Layer] { + let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; + for shard in shards.flatten() { + let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; + for f in files.flatten() { + let p = f.path(); + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if name.contains(".tmp") { + // A writer that died mid-write; old temporaries are removed. + if f.metadata().and_then(|m| m.modified()).is_ok_and(|t| t.elapsed().is_ok_and(|e| e.as_secs() > 600)) { + let _ = std::fs::remove_file(&p); + } + continue; + } + let Some(hex) = name.strip_suffix(".ecc") else { continue }; + let Ok(key) = u128::from_str_radix(hex, 16) else { continue }; + let Ok(md) = f.metadata() else { continue }; + found.push((md.modified().unwrap_or(std::time::UNIX_EPOCH), kind, key, md.len())); + } + } + } + found.sort_by_key(|e| e.0); + let mut idx = Index::default(); + for (_, kind, key, bytes) in found { + idx.clock += 1; + let clock = idx.clock; + idx.total += bytes; + idx.map.insert((kind, key), Meta { bytes, last_use: clock }); + } + if let Ok(mut g) = self.index.lock() { + *g = idx; + } + } + + fn path(&self, kind: Kind, key: u128) -> PathBuf { + let hex = format!("{key:032x}"); + self.dir.join(kind.dir()).join(&hex[..2]).join(format!("{hex}.ecc")) + } + + pub fn max_bytes(&self) -> u64 { + self.max_bytes.load(Ordering::Relaxed) + } + + /// Change the size limit (evicts the least recently used entries when shrinking). + pub fn set_max_bytes(&self, b: u64) { + self.max_bytes.store(b, Ordering::Relaxed); + self.evict(); + } + + /// Layer buffers faster to render than this are not worth a disk write. + pub fn set_min_layer_ms(&self, ms: u64) { + self.min_layer_ms.store(ms, Ordering::Relaxed); + } + + pub fn min_layer_ms(&self) -> u64 { + self.min_layer_ms.load(Ordering::Relaxed) + } + + pub fn contains(&self, kind: Kind, key: u128) -> bool { + self.index.lock().is_ok_and(|g| g.map.contains_key(&(kind, key))) + } + + /// Keys of one kind (unordered). + pub fn keys(&self, kind: Kind) -> Vec { + self.index.lock().map(|g| g.map.keys().filter(|k| k.0 == kind).map(|k| k.1).collect()).unwrap_or_default() + } + + pub fn stats(&self) -> DiskStats { + let (entries, bytes) = self.index.lock().map(|g| (g.map.len(), g.total)).unwrap_or_default(); + DiskStats { + entries, + bytes, + max_bytes: self.max_bytes(), + hits: self.hits.load(Ordering::Relaxed), + misses: self.misses.load(Ordering::Relaxed), + writes: self.writes.load(Ordering::Relaxed), + evictions: self.evictions.load(Ordering::Relaxed), + dropped: self.dropped.load(Ordering::Relaxed), + } + } + + /// Delete every entry (Edit ▸ Purge ▸ All Disk Cache / Empty Disk Cache). + pub fn clear(&self) { + self.flush(); + if let Ok(mut g) = self.index.lock() { + *g = Index::default(); + } + for kind in [Kind::Frame, Kind::Layer] { + let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); + } + } + + /// Wait until queued writes are on disk. + pub fn flush(&self) { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + #[cfg(target_arch = "wasm32")] + { + q.jobs.clear(); + } + while !q.jobs.is_empty() || q.busy { + q = match cv.wait(q) { + Ok(q) => q, + Err(_) => return, + }; + } + } + + fn read(&self, kind: Kind, key: u128) -> Option> { + if !self.contains(kind, key) { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + } + let path = self.path(kind, key); + let data = std::fs::read(&path).ok(); + let ok = data.as_deref().and_then(|d| verify(d, kind)); + match ok { + Some(payload) => { + let payload = payload.to_vec(); + self.hits.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + if let Some(m) = g.map.get_mut(&(kind, key)) { + m.last_use = c; + } + } + // Persist the recency for the next session (best effort). + if let Ok(f) = std::fs::File::options().append(true).open(&path) { + let _ = f.set_modified(std::time::SystemTime::now()); + } + Some(payload) + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); + // Gone (another process evicted it) or damaged: forget it. + if data.is_some() { + let _ = std::fs::remove_file(&path); + } + self.forget(kind, key); + None + } + } + } + + fn forget(&self, kind: Kind, key: u128) { + if let Ok(mut g) = self.index.lock() + && let Some(m) = g.map.remove(&(kind, key)) + { + g.total -= m.bytes; + } + } + + fn enqueue(&self, kind: Kind, key: u128, payload: Vec) { + #[cfg(target_arch = "wasm32")] + { + self.write_now(kind, key, &payload); + } + #[cfg(not(target_arch = "wasm32"))] + { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + if q.jobs.len() >= QUEUE_LIMIT { + self.dropped.fetch_add(1, Ordering::Relaxed); + return; + } + q.jobs.push_back(Job::Write(kind, key, payload)); + cv.notify_all(); + } + } + + /// Write an entry now (atomic rename), then evict down to the limit. + fn write_now(&self, kind: Kind, key: u128, payload: &[u8]) { + if self.contains(kind, key) { + return; + } + let path = self.path(kind, key); + let Some(parent) = path.parent() else { return }; + if std::fs::create_dir_all(parent).is_err() { + return; + } + let mut data = Vec::with_capacity(payload.len() + 16); + data.extend_from_slice(MAGIC); + data.push(kind.tag()); + data.extend_from_slice(&[0, 0, 0]); + data.extend_from_slice(payload); + let sum = checksum(&data); + data.extend_from_slice(&sum.to_le_bytes()); + let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!("{key:032x}.tmp.{}.{n}", std::process::id())); + if std::fs::write(&tmp, &data).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + if std::fs::rename(&tmp, &path).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + self.writes.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + let bytes = data.len() as u64; + if let Some(old) = g.map.insert((kind, key), Meta { bytes, last_use: c }) { + g.total -= old.bytes; + } + g.total += bytes; + } + self.evict(); + } + + /// Remove least recently used entries until the cache fits its limit. + fn evict(&self) { + let max = self.max_bytes(); + let victims: Vec<(Kind, u128)> = { + let Ok(mut g) = self.index.lock() else { return }; + if g.total <= max { + return; + } + let mut by_age: Vec<(u64, (Kind, u128), u64)> = g.map.iter().map(|(k, m)| (m.last_use, *k, m.bytes)).collect(); + by_age.sort_unstable_by_key(|e| e.0); + let mut out = vec![]; + for (_, k, bytes) in by_age { + if g.total <= max { + break; + } + g.map.remove(&k); + g.total -= bytes; + out.push(k); + } + out + }; + for (kind, key) in victims { + let _ = std::fs::remove_file(self.path(kind, key)); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + } + + // ------------------------------------------------------------ typed entries + + pub fn get_frame(&self, key: u128) -> Option { + let p = self.read(Kind::Frame, key)?; + decode_frame(&p) + } + + /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). + pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 { + return; + } + self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)); + } + + pub fn get_layer(&self, key: u128) -> Option { + let p = self.read(Kind::Layer, key)?; + decode_layer(&p) + } + + /// Queue a layer buffer for writing. + pub fn put_layer(&self, key: u128, buf: &Buf) { + if self.contains(Kind::Layer, key) { + return; + } + self.enqueue(Kind::Layer, key, encode_layer(buf)); + } + + /// Write a layer buffer synchronously (tests). + pub fn put_layer_now(&self, key: u128, buf: &Buf) { + self.write_now(Kind::Layer, key, &encode_layer(buf)); + } + + /// Write a frame synchronously (tests). + pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { + let (m, cv) = &*q; + loop { + let job = { + let Ok(mut g) = m.lock() else { return }; + loop { + if let Some(j) = g.jobs.pop_front() { + g.busy = true; + break j; + } + if dc.strong_count() == 0 { + return; + } + g = match cv.wait_timeout(g, std::time::Duration::from_millis(500)) { + Ok((g, _)) => g, + Err(_) => return, + }; + } + }; + let Some(cache) = dc.upgrade() else { return }; + match job { + Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload), + } + drop(cache); + if let Ok(mut g) = m.lock() { + g.busy = false; + } + cv.notify_all(); + } +} + +/// Check magic, kind and checksum; returns the payload. +fn verify(d: &[u8], kind: Kind) -> Option<&[u8]> { + if d.len() < 16 || &d[..4] != MAGIC || d[4] != kind.tag() { + return None; + } + let (body, sum) = d.split_at(d.len() - 8); + let mut s = [0u8; 8]; + s.copy_from_slice(sum); + (checksum(body) == u64::from_le_bytes(s)).then_some(&body[8..]) +} + +fn encode_frame(w: u32, h: u32, rgba: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgba.len() / 2 + 16); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(rgba)); + out +} + +fn decode_frame(p: &[u8]) -> Option { + if p.len() < 8 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let rgba = lz4_flex::decompress_size_prepended(&p[8..]).ok()?; + (rgba.len() == w as usize * h as usize * 4).then_some(Frame8 { width: w, height: h, rgba }) +} + +fn encode_layer(b: &Buf) -> Vec { + let (w, h) = (b.img.width, b.img.height); + let n = b.img.data.len(); + // Byte planes: all first bytes of every float, then all second bytes… (compresses well). + let mut planes = vec![0u8; n * 16]; + for (i, px) in b.img.data.iter().enumerate() { + for c in 0..4 { + let bytes = px[c].to_le_bytes(); + for (k, byte) in bytes.iter().enumerate() { + planes[(k * 4 + c) * n + i] = *byte; + } + } + } + let mut out = Vec::with_capacity(planes.len() / 4 + 40); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&b.offset[0].to_le_bytes()); + out.extend_from_slice(&b.offset[1].to_le_bytes()); + out.extend_from_slice(&b.scale.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(&planes)); + out +} + +fn decode_layer(p: &[u8]) -> Option { + if p.len() < 32 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let f = |i: usize| -> Option { Some(f64::from_le_bytes(p[i..i + 8].try_into().ok()?)) }; + let offset = [f(8)?, f(16)?]; + let scale = f(24)?; + let planes = lz4_flex::decompress_size_prepended(&p[32..]).ok()?; + let n = w as usize * h as usize; + if planes.len() != n * 16 { + return None; + } + let mut img = Image::new(w, h); + for (i, px) in img.data.iter_mut().enumerate() { + for c in 0..4 { + let mut bytes = [0u8; 4]; + for (k, byte) in bytes.iter_mut().enumerate() { + *byte = planes[(k * 4 + c) * n + i]; + } + px[c] = f32::from_le_bytes(bytes); + } + } + Some(Buf { img, offset, scale }) +} + +// ---------------------------------------------------------------- content keys + +/// Bump when rendering changes in a way that makes old disk entries wrong. +const RENDER_VERSION: &str = "effectcraft-render-1"; + +fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { + h.write(format!("{v:?}").as_bytes()); + h.write(&[0xff]); +} + +/// A file's size and modification time (so edited footage gets new keys). +fn file_stamp(h: &mut Hash128, path: &str) { + if path.is_empty() { + return; + } + if let Ok(md) = std::fs::metadata(path) { + h.write_u64(md.len()); + if let Ok(t) = md.modified() + && let Ok(d) = t.duration_since(std::time::UNIX_EPOCH) + { + h.write_u64(d.as_nanos() as u64); + } + } +} + +fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) { + file_stamp(h, &f.path); + for p in &f.sequence { + file_stamp(h, p); + } +} + +/// Salt for layer cache keys on disk: the renderer version and every footage file's identity +/// (layer keys name footage items, not file contents). +pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + for (id, it) in &project.items { + if let effectcraft_project::ItemKind::Footage(f) = &it.kind { + h.write_u64(id.0); + hash_debug(&mut h, f); + footage_stamp(&mut h, f); + } + } + let v = h.finish(); + (v as u64) ^ ((v >> 64) as u64) +} + +/// Content key of a composition's pixels: project render settings, the comp and everything it +/// uses (nested comps, footage, solids, footage files' size and time). With expressions in play +/// (which can read any comp) the whole project counts. +pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { + use effectcraft_project::{ItemKind, LayerSource}; + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + hash_debug(&mut h, &project.settings); + let mut seen = std::collections::BTreeSet::new(); + let mut stack = vec![comp]; + let mut expressions = false; + while let Some(id) = stack.pop() { + if !seen.insert(id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { + stack.push(item); + } + if !expressions { + l.props.walk("", &mut |_, p| expressions |= p.expr.is_some()); + } + } + } + } + let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; + for id in ids { + let Some(it) = project.item(id) else { continue }; + h.write_u64(id.0); + h.write(it.name.as_bytes()); + hash_debug(&mut h, &it.kind); + if let ItemKind::Footage(f) = &it.kind { + footage_stamp(&mut h, f); + } + } + h.write_u64(comp.0); + h.finish() +} + +/// Disk key of one viewer frame. +pub fn frame_key(content: u128, frame: i64, scale_milli: u32, view: u64) -> u128 { + let mut h = Hash128::default(); + h.write(&content.to_le_bytes()); + h.write_u64(frame as u64); + h.write_u64(scale_milli as u64); + h.write_u64(view); + h.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmpdir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-disk-cache-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + fn buf(seed: u32, w: u32, h: u32) -> Buf { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + let v = ((i as u32).wrapping_mul(2654435761u32).wrapping_add(seed) % 1000) as f32 / 999.0; + *p = [v, v * 0.5, 1.0 - v, if i % 7 == 0 { 0.0 } else { 1.0 }]; + } + Buf { img, offset: [1.5, -2.25], scale: 0.5 } + } + + #[test] + fn hit_miss_and_exact_round_trip() { + let dir = tmpdir("hit"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert!(dc.get_layer(42).is_none()); + let b = buf(1, 33, 17); + dc.put_layer(42, &b); + dc.flush(); + let back = dc.get_layer(42).unwrap(); + assert_eq!(back.img, b.img); + assert_eq!((back.offset, back.scale), (b.offset, b.scale)); + let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); + dc.put_frame(7, 20, 10, &rgba); + dc.flush(); + assert_eq!(dc.get_frame(7).unwrap().rgba, rgba); + let st = dc.stats(); + assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2)); + // Wrong kind is a miss. + assert!(dc.get_frame(42).is_none()); + } + + #[test] + fn persists_across_restarts_and_clears() { + let dir = tmpdir("persist"); + { + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_layer_now(1, &buf(1, 8, 8)); + dc.put_frame_now(2, 2, 2, &[9; 16]); + } + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.stats().entries, 2); + assert!(dc.contains(Kind::Layer, 1) && dc.contains(Kind::Frame, 2)); + assert_eq!(dc.get_frame(2).unwrap().rgba, vec![9; 16]); + dc.clear(); + assert_eq!(dc.stats().entries, 0); + assert!(dc.get_frame(2).is_none()); + let again = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(again.stats().entries, 0); + } + + #[test] + fn lru_eviction_respects_the_limit() { + let dir = tmpdir("lru"); + let dc = DiskCache::open(&dir, u64::MAX).unwrap(); + for k in 0..6u128 { + dc.put_layer_now(k, &buf(k as u32, 64, 64)); + } + let per = dc.stats().bytes / 6; + // Touch 0 so it is the most recently used. + assert!(dc.get_layer(0).is_some()); + dc.set_max_bytes(per * 3 + per / 2); + let st = dc.stats(); + assert!(st.bytes <= per * 3 + per / 2, "{st:?}"); + assert_eq!(st.entries, 3); + assert!(dc.contains(Kind::Layer, 0), "recently used survives"); + assert!(!dc.contains(Kind::Layer, 1) && !dc.contains(Kind::Layer, 2) && !dc.contains(Kind::Layer, 3)); + assert_eq!(st.evictions, 3); + // Files are really gone. + let files = walk(&dir); + assert_eq!(files, 3); + // New writes keep evicting. + dc.put_layer_now(99, &buf(9, 64, 64)); + assert!(dc.stats().bytes <= per * 3 + per / 2); + } + + fn walk(d: &Path) -> usize { + let mut n = 0; + if let Ok(rd) = std::fs::read_dir(d) { + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + n += walk(&p); + } else if p.extension().is_some_and(|x| x == "ecc") { + n += 1; + } + } + } + n + } + + #[test] + fn damaged_files_are_misses() { + let dir = tmpdir("damaged"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_frame_now(5, 4, 4, &[1; 64]); + let p = dc.path(Kind::Frame, 5); + let mut d = std::fs::read(&p).unwrap(); + let n = d.len(); + d[n / 2] ^= 0xff; + std::fs::write(&p, d).unwrap(); + assert!(dc.get_frame(5).is_none()); + assert!(!dc.contains(Kind::Frame, 5)); + assert!(!p.exists()); + } + + #[test] + fn concurrent_readers_and_writers() { + let dir = tmpdir("concurrent"); + let a = DiskCache::open(&dir, 1 << 30).unwrap(); + // A second instance on the same folder (another process, in effect). + let b = DiskCache::open(&dir, 1 << 30).unwrap(); + std::thread::scope(|s| { + for t in 0..4u32 { + let (a, b) = (a.clone(), b.clone()); + s.spawn(move || { + for i in 0..20u128 { + let k = i % 5; + let w = if t % 2 == 0 { &a } else { &b }; + w.put_layer_now(k, &buf(k as u32, 16, 16)); + if let Some(x) = w.get_layer(k) { + assert_eq!(x.img, buf(k as u32, 16, 16).img); + } + } + }); + } + }); + b.rescan(); + for k in 0..5u128 { + assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 14a81575f..a4e718a30 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio; pub mod cache; pub mod color; +pub mod disk_cache; use color::Region; pub mod eval; pub mod masks; @@ -46,6 +47,16 @@ pub trait FootageSource: Send + Sync { fn cache_budget(&self) -> Option { None } + /// Vector footage (SVG) rasterised at `scale` × its pixel size, for Continuously Rasterize. + /// `None` when the footage is not vector or cannot be read. + fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option> { + None + } +} + +/// Footage that can be rasterised at any scale (SVG). +pub fn is_vector_footage(f: &Footage) -> bool { + f.codec == "SVG" } /// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]). @@ -348,7 +359,12 @@ impl<'a> Renderer<'a> { /// stays sharp when scaled up). pub fn raster_scale(&self, ctx: &EvalCtx, layer: &Layer) -> f64 { let s = self.opts.scale; - if !layer.switches.collapse || !matches!(layer.source, LayerSource::Text | LayerSource::Shape) { + let vector = match layer.source { + LayerSource::Text | LayerSource::Shape => true, + LayerSource::Footage { item } => matches!(self.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if is_vector_footage(f)), + _ => false, + }; + if !layer.switches.collapse || !vector { return s; } let (l2c, _) = ctx.layer_to_comp(layer); @@ -631,6 +647,17 @@ impl<'a> Renderer<'a> { if !f.has_video { return None; } + // Continuously rasterised vector footage: drawn at the on-screen scale. + if layer.switches.collapse && is_vector_footage(f) { + let k = self.raster_scale(ctx, layer); + if let Some(img) = self.footage.vector_frame(*item, f, k) { + let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + return Some(buf); + } + } let lt = ctx.source_time(layer); let img = self.footage_frame(ctx, layer, *item, f, lt)?; let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 }; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index dfd49ccc3..991fac323 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -51,6 +51,7 @@ fn audio_footage() -> Footage { missing: false, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/render/src/tests_color.rs b/crates/render/src/tests_color.rs index 87f9dd1ae..016cdc932 100644 --- a/crates/render/src/tests_color.rs +++ b/crates/render/src/tests_color.rs @@ -172,6 +172,7 @@ fn footage(profile: Option) -> Footage { missing: false, sequence: vec![], color_profile: profile, + layer: None, } } diff --git a/crates/render/src/tests_frame_blend.rs b/crates/render/src/tests_frame_blend.rs index c172b81c8..3c301d38c 100644 --- a/crates/render/src/tests_frame_blend.rs +++ b/crates/render/src/tests_frame_blend.rs @@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None); diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml new file mode 100644 index 000000000..0e1af7719 --- /dev/null +++ b/crates/svg/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "effectcraft-svg" +description = "SVG import: parses SVG 1.1 / SVG 2 static vector content (paths, basic shapes, groups, transforms, use, CSS styles, solid and gradient fills/strokes, opacity) and rasterises it at any scale" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-path = { workspace = true } +kurbo = { workspace = true } +roxmltree = "0.21" +rayon = { workspace = true } diff --git a/crates/svg/README.md b/crates/svg/README.md new file mode 100644 index 000000000..862bd01d0 --- /dev/null +++ b/crates/svg/README.md @@ -0,0 +1,26 @@ +# effectcraft-svg + +SVG import for EffectCraft: `.svg` files import as footage (rasterised at any scale, so +Continuously Rasterize stays sharp) and Layer ▸ Create ▸ Create Shapes from Vector Layer turns +them into editable shape layers. + +Implemented from the W3C **Scalable Vector Graphics (SVG) 1.1 (Second Edition)** recommendation +(16 August 2011) and the **SVG 2** candidate recommendation (4 October 2018) for the parts listed +here, plus **CSS Color Module Level 3** for colour keywords and functions: + +- document: `svg` (width/height, `viewBox`, `preserveAspectRatio`), nested `svg`, `g`, `defs`, + `use` (x/y, `href` / `xlink:href`), `symbol` (through `use`), `title`/`desc`/`metadata` ignored; +- shapes: `path` (all path data commands incl. elliptical arcs), `rect` (rx/ry), `circle`, + `ellipse`, `line`, `polyline`, `polygon`; +- `transform` lists (matrix, translate, scale, rotate with centre, skewX, skewY); +- painting: `fill`, `fill-rule`, `fill-opacity`, `stroke`, `stroke-width`, `stroke-linecap`, + `stroke-linejoin`, `stroke-miterlimit`, `stroke-dasharray`, `stroke-dashoffset`, + `stroke-opacity`, `opacity`, `display`, `visibility`, `color`/`currentColor`; +- `linearGradient` and `radialGradient` (with `fx`/`fy` focal point), `gradientUnits`, + `gradientTransform`, `spreadMethod`, `href` inheritance of stops and attributes; +- styling through presentation attributes, the `style` attribute and ` + + + + + + + + + + + + + + + + + diff --git a/crates/svg/tests/fixtures/paths-gradients.svg.attribution b/crates/svg/tests/fixtures/paths-gradients.svg.attribution new file mode 100644 index 000000000..d4b8758fe --- /dev/null +++ b/crates/svg/tests/fixtures/paths-gradients.svg.attribution @@ -0,0 +1,7 @@ +asset: paths-gradients.svg +title: SVG test fixture: path data, gradients, CSS styles and use +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import) +notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests. diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 62239f191..d0e5a6d93 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -7,6 +7,11 @@ //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. //! +//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also +//! written to disk under a content key (the comp's content hash, frame, scale, view and render +//! options), and a job first looks there: frames survive restarts and project reopenings. The +//! timeline shows disk-only frames as a blue cache bar. +//! //! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set //! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer //! registers with egui-wgpu and draws directly; pixels are read back only when something needs @@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, Gpu}; @@ -122,6 +128,37 @@ pub struct RenderSource { /// Show Channel / exposure / region-of-interest drawing read the CPU frame texture, so GPU /// frames are read back (the compositing still runs on the GPU). pub gpu_display: bool, + /// The persistent disk cache, when enabled. + pub disk: Option>, +} + +/// Content keys of (project revision, comp), computed once per revision. +type ContentKeys = Arc>>; + +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { + return k; + } + let k = disk_cache::comp_content_key(project, ItemId(comp)); + if let Ok(mut m) = keys.lock() { + if m.len() > 256 { + m.clear(); + } + m.insert((revision, comp), k); + } + k +} + +fn opts_hash(o: &RenderOpts) -> u64 { + let mut h = disk_cache::Hash128::default(); + h.write(format!("{o:?}").as_bytes()); + h.finish() as u64 +} + +/// Disk key of a viewer frame (render options included: region of interest, draft…). +fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { + let content = content_key(keys, project, key.revision, key.comp); + disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } pub struct Frames { @@ -135,6 +172,7 @@ pub struct Frames { ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, + content_keys: ContentKeys, } impl Default for Frames { @@ -149,6 +187,7 @@ impl Default for Frames { pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"), ctx: None, last_ms: Arc::new(Mutex::new(0.0)), + content_keys: Arc::default(), } } } @@ -195,6 +234,21 @@ impl Frames { v } + /// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar. + /// `opts` are the viewer's render options (part of the disk key). + pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec { + let Some(dc) = &src.disk else { return vec![] }; + let ram: HashSet = self.cached_frames(revision, comp, scale).into_iter().collect(); + let oh = opts_hash(opts); + (0..frames.clamp(0, 100_000)) + .filter(|f| !ram.contains(f)) + .filter(|f| { + let key = FrameKey { revision, comp, frame: *f, scale, view }; + dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh)) + }) + .collect() + } + /// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest /// frames when it shrinks. pub fn set_budget(&self, bytes: usize) { @@ -278,7 +332,14 @@ impl Frames { } fn worker(&self) -> Worker { - Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() } + Worker { + queue: self.queue.clone(), + cache: self.cache.clone(), + inflight: self.inflight.clone(), + ctx: self.ctx.clone(), + last: self.last_ms.clone(), + content_keys: self.content_keys.clone(), + } } /// Render queued frames on this thread, most important first, until the queue is empty or @@ -306,12 +367,13 @@ struct Worker { inflight: Arc>>, ctx: Option, last: Arc>, + content_keys: ContentKeys, } impl Worker { /// Render the queued job that matters most right now. `false` when the queue was empty. fn run_next(&self) -> bool { - let Worker { queue, cache, inflight, ctx, last } = self; + let Worker { queue, cache, inflight, ctx, last, content_keys } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; // Urgent first (newest), then prefetch in request order. @@ -325,19 +387,18 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); - r.expr = job.src.expr.as_deref(); - r.cache = Some(&job.src.layer_cache); - r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); - // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no - // adapter, or a frame the GPU cannot finish here) the CPU renders it. - let gpu_frame = match (&job.src.gpu, r.active_accel()) { - (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), - _ => None, - }; - let ci = match gpu_frame { - Some(f) => FrameImage::Gpu(Arc::new(f)), - None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); + let ci = match from_disk { + Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))), + None => { + let ci = self.render_job(&job); + if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba); + } + ci + } }; if job.urgent && let Ok(mut l) = last.lock() @@ -361,6 +422,23 @@ impl Worker { } true } + + fn render_job(&self, job: &Job) -> FrameImage { + let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); + r.expr = job.src.expr.as_deref(); + r.cache = Some(&job.src.layer_cache); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), + _ => None, + }; + match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + } + } } struct Job { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..ec6a2278a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -308,6 +308,7 @@ impl EffectcraftApp { layer_cache: self.session.layer_cache.clone(), gpu: self.gpu.clone(), gpu_display: false, + disk: self.session.disk_cache.clone(), } } @@ -405,13 +406,11 @@ impl EffectcraftApp { self.request_frame_with(comp, frame, scale, true); } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { - let Some(c) = self.session.project.comp(comp) else { return }; - let key = self.frame_key(comp, frame, scale); - let t = c.frame_rate.tick_of(frame); + /// Render options of viewer frames of `comp` at `scale`. + pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); - let opts = RenderOpts { + RenderOpts { scale, motion_blur: true, guides: true, @@ -419,7 +418,14 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, - }; + } + } + + fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + let Some(c) = self.session.project.comp(comp) else { return }; + let key = self.frame_key(comp, frame, scale); + let t = c.frame_rate.tick_of(frame); + let opts = self.frame_opts(comp, scale); if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2d..7ebadbf41 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -716,7 +716,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, @@ -739,6 +739,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option bool { } /// If `id` is a dialog command invoked without its parameters, open its form and return true. +/// The import includes a Photoshop document. +fn is_layered(p: &Value) -> bool { + let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb"))); + match p.get("paths").or(p.get("path")) { + Some(Value::Array(a)) => a.iter().any(psd), + Some(v) => psd(v), + None => false, + } +} + pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let s = &app.session; let comp = s.active_comp(); @@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t); let base = p.clone(); let (title, fields): (String, Vec) = match id { + // Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes). + "file.import" if !has(p, &["importAs"]) && is_layered(p) => ( + "Import Photoshop File".into(), + vec![Field::choice( + "importAs", + "Import Kind", + &[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))], + 1, + )], + ), "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5b6c085f5..785491d77 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -602,6 +602,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((a, b)) = run { draw_run(a, b); } + // Disk-cached frames not in RAM (blue), like After Effects. + if app.session.disk_cache.is_some() { + let frames = comp.frame_rate.frame_at(comp.duration); + let opts = app.frame_opts(cid, scale_key as f64 / 1000.0); + let disk = app.frames.disk_frames(&app.render_source(), app.session.revision, cid.0, scale_key, app.view_hash(cid), frames, &opts); + let draw_blue = |a: i64, b: i64| { + let x0 = tm.x(a as f64 * fd); + let x1 = tm.x((b + 1) as f64 * fd); + p.rect_filled(Rect::from_min_max(pos2(x0, cy0), pos2(x1, cy0 + 2.5)), 0.0, t.cache_blue); + }; + let mut run: Option<(i64, i64)> = None; + for f in disk { + run = match run { + Some((a, b)) if f == b + 1 => Some((a, f)), + Some((a, b)) => { + draw_blue(a, b); + Some((f, f)) + } + None => Some((f, f)), + }; + } + if let Some((a, b)) = run { + draw_blue(a, b); + } + } // Ticks + labels. // Label spacing in whole frames (AE: `00:15f`-style seconds:frames labels). let fps_i = fr.as_f64().round().max(1.0) as i64; diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 5c82e9168..c32ecb248 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -60,6 +60,8 @@ pub struct Tokens { pub cti: Color32, pub work_area: Color32, pub cache_green: Color32, + /// Timeline cache bar for frames held in the disk cache (not in RAM). + pub cache_blue: Color32, pub keyframe: Color32, pub keyframe_selected: Color32, pub pasteboard: Color32, @@ -108,6 +110,7 @@ impl Tokens { cti: Color32::from_rgb(0x2d, 0x8c, 0xeb), work_area: Color32::from_rgb(0x4a, 0x4a, 0x4a), cache_green: Color32::from_rgb(0x3c, 0xa6, 0x4c), + cache_blue: Color32::from_rgb(0x3a, 0x6f, 0xd8), keyframe: Color32::from_rgb(0xa8, 0xa8, 0xa8), keyframe_selected: Color32::from_rgb(0x3d, 0x8f, 0xf5), pasteboard: Color32::from_rgb(0x1a, 0x1a, 0x1a), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3d..09e150474 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,10 +27,12 @@ const LAYERS: &[(&str, u8)] = &[ ("raster", 1), ("keyframe", 1), ("path", 1), + ("psd", 1), ("project", 2), ("text", 2), ("effects", 2), ("track", 2), + ("svg", 2), ("render", 3), ("media", 3), ("expr", 3), From 78cd313d3cebfbcca543a260cb5ca52840d28c0e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:13:09 -0400 Subject: [PATCH 04/14] M14.4: scripting with an After Effects-style JavaScript object model New crate effectcraft-script (L4): boa runtime with app/Project/CompItem/ Layer/PropertyGroup/Property/TextDocument/MarkerValue/RenderQueue/File object model over engine commands (undoable, undo groups), AE match-name table, file/network security gate. Entry points: script.run command, File > Scripts > Run Script File (.jsx/.js), Window > Script Console, effectcraft-cli script, MCP run_script. Engine: project.delete/duplicate, prop.renameGroup/setGroupEnabled/removeGroup/duplicateGroup/moveGroup. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 17 + Cargo.toml | 1 + apps/effectcraft-cli/src/main.rs | 51 +- apps/effectcraft-cli/tests/cli.rs | 31 + apps/effectcraft-web/src/lib.rs | 5 +- crates/automation/Cargo.toml | 3 + crates/automation/src/tests.rs | 28 + crates/automation/src/tools.rs | 24 + crates/engine/src/commands/file_more.rs | 60 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/project_items.rs | 96 ++ crates/engine/src/commands/prop_groups.rs | 107 ++ crates/engine/src/commands/text_anim.rs | 3 +- crates/engine/src/lib.rs | 18 + crates/engine/src/menus.rs | 1 + crates/engine/src/tests_project_items.rs | 57 + crates/host/Cargo.toml | 3 +- crates/host/src/lib.rs | 38 +- crates/script/Cargo.toml | 27 + crates/script/src/lib.rs | 64 + crates/script/src/matchnames.rs | 443 +++++ crates/script/src/model.rs | 679 ++++++++ crates/script/src/prelude.js | 1689 +++++++++++++++++++ crates/script/src/runtime.rs | 546 ++++++ crates/script/src/tests.rs | 392 +++++ crates/ui-egui/src/dock.rs | 5 +- crates/ui-egui/src/menus.rs | 2 +- crates/ui-egui/src/panels/mod.rs | 2 + crates/ui-egui/src/panels/project.rs | 14 +- crates/ui-egui/src/panels/script_console.rs | 144 ++ crates/ui-egui/src/state.rs | 4 + crates/ui-egui/tests/ui_script_console.rs | 44 + docs/agents.md | 14 + docs/architecture.md | 28 +- docs/parity.md | 4 +- xtask/src/main.rs | 3 + 36 files changed, 4621 insertions(+), 28 deletions(-) create mode 100644 crates/engine/src/commands/prop_groups.rs create mode 100644 crates/script/Cargo.toml create mode 100644 crates/script/src/lib.rs create mode 100644 crates/script/src/matchnames.rs create mode 100644 crates/script/src/model.rs create mode 100644 crates/script/src/prelude.js create mode 100644 crates/script/src/runtime.rs create mode 100644 crates/script/src/tests.rs create mode 100644 crates/ui-egui/src/panels/script_console.rs create mode 100644 crates/ui-egui/tests/ui_script_console.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..c1e86483b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1145,6 +1145,7 @@ name = "effectcraft-automation" version = "0.1.0" dependencies = [ "effectcraft-engine", + "effectcraft-script", "image", "serde_json", "thiserror 2.0.21", @@ -1287,6 +1288,7 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-script", "serde_json", ] @@ -1392,6 +1394,21 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-script" +version = "0.1.0" +dependencies = [ + "boa_engine", + "effectcraft-engine", + "effectcraft-expr", + "effectcraft-keyframe", + "effectcraft-path", + "effectcraft-project", + "effectcraft-render", + "effectcraft-time", + "serde_json", +] + [[package]] name = "effectcraft-text" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf45..6c5ac99ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-script = { path = "crates/script" } # foundation thiserror = "2" diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66c..b8222ddcf 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -15,6 +15,7 @@ //! effectcraft-cli render F.ecproj --queue render the project's Render Queue //! effectcraft-cli bench [--comp C] [--time S] [--scale K] [--n N] [--play N] [--gpu] render timings //! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half) +//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script //! effectcraft-cli mcp [--bridge PORT] MCP server on stdio //! //! Project: --project F.ecproj (or a positional *.ecproj) | --demo | --empty (default: demo; mcp: empty) @@ -32,7 +33,7 @@ use effectcraft_automation::tools::{self, Reply}; use effectcraft_automation::{Backend, McpServer}; use serde_json::{Value, json}; -const USAGE: &str = "usage: effectcraft-cli [args] [--json] +const USAGE: &str = "usage: effectcraft-cli [args] [--json] info project + engine summary commands [--filter TEXT] [--enabled] list engine commands exec [--params JSON] run one engine command @@ -47,6 +48,9 @@ const USAGE: &str = "usage: effectcraft-cli : id or name, '-' = active comp; : id, '#n' or name; : JSON or bare string"; @@ -68,6 +72,7 @@ const VALUED: &[&str] = &[ "--out", "--expression", "--depth", + "--eval", // render "--format", "--start", @@ -243,6 +248,49 @@ fn tool(b: &mut Backend, name: &str, a: Value) -> Result { } } +/// `script FILE.jsx [project]` / `script --eval CODE`: run JavaScript against a headless session +/// (an empty project unless one is given, or `--demo`) or the running app (`--bridge`). +fn script_cmd(args: &Args, json_out: bool) -> Result<(), Failure> { + let (code, name) = match (args.opt("--eval"), args.pos.first()) { + (Some(c), _) => (c.to_string(), "eval".to_string()), + (None, Some(f)) => (std::fs::read_to_string(f).map_err(|e| Failure::Error(format!("cannot read {f}: {e}")))?, f.clone()), + (None, None) => return usage_err("script needs a .jsx/.js file or --eval CODE"), + }; + let mut b = backend(args, false)?; + let r = b.exec("script.run", json!({"code": code, "name": name}))?; + let saved = maybe_save(&mut b, args)?; + if json_out { + let mut o = r.clone(); + if let Some(p) = &saved { + o["saved"] = json!(p); + } + emit(&o, true); + } else { + if let Some(out) = r["output"].as_str().filter(|o| !o.is_empty()) { + println!("{out}"); + } + if !r["result"].is_null() && r["error"].is_null() { + println!("{}", r["result"]); + } + if let Some(p) = saved { + eprintln!("saved {p}"); + } + } + if let Some(e) = r.get("error").filter(|e| !e.is_null()) { + let pos = match (e["line"].as_u64(), e["column"].as_u64()) { + (Some(l), Some(c)) => format!(":{l}:{c}"), + (Some(l), None) => format!(":{l}"), + _ => String::new(), + }; + let msg = format!("{}{pos}: {}", e["file"].as_str().unwrap_or(&name), e["message"].as_str().unwrap_or("script error")); + if json_out { + std::process::exit(1); + } + return Err(Failure::Error(msg)); + } + Ok(()) +} + /// Print a result: compact JSON with `--json`, else pretty JSON. fn emit(v: &Value, json_out: bool) { if json_out { @@ -315,6 +363,7 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> { } } } + "script" => script_cmd(args, json_out)?, "exec" => { let Some(id) = args.pos.first().cloned() else { return usage_err("exec needs a command id") }; let params = match (args.opt("--params"), args.pos.get(1).map(String::as_str)) { diff --git a/apps/effectcraft-cli/tests/cli.rs b/apps/effectcraft-cli/tests/cli.rs index c84277202..d7c48762f 100644 --- a/apps/effectcraft-cli/tests/cli.rs +++ b/apps/effectcraft-cli/tests/cli.rs @@ -131,3 +131,34 @@ fn mcp_over_stdio() { assert_eq!(replies[2]["result"]["isError"], false, "{}", replies[2]); assert_eq!(replies[3]["result"]["content"][0]["type"], "image"); } + +#[test] +fn script_file_and_eval() { + let proj = tmp("scripted.ecproj"); + let p = proj.to_str().unwrap(); + let jsx = tmp("build.jsx"); + std::fs::write( + &jsx, + "app.beginUndoGroup('Build');\nvar c = app.project.items.addComp('FromJsx', 160, 90, 1, 2, 24);\nc.layers.addSolid([1, 0, 0], 'Red', 160, 90, 1);\napp.endUndoGroup();\nwriteLn('built ' + c.name);\nc.numLayers", + ) + .unwrap(); + let v = ok_json(&["script", jsx.to_str().unwrap(), "--save-as", p]); + assert_eq!(v["ok"], json!(true), "{v}"); + assert_eq!(v["result"], json!(1)); + assert_eq!(v["output"], json!("built FromJsx")); + assert_eq!(v["saved"], p); + // Run against the saved project (positional), plain output. + let out = bin().args(["script", "--eval", "app.project.item(1).name + ' has ' + app.project.item(1).numLayers", p]).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "\"FromJsx has 1\""); + // Errors: exit 1 with file:line:col. + let bad = tmp("bad.jsx"); + std::fs::write(&bad, "var a = 1;\n\nmissingFunction();\n").unwrap(); + let out = bin().args(["script", bad.to_str().unwrap()]).output().unwrap(); + assert_eq!(out.status.code(), Some(1)); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains("bad.jsx:3:1") && err.contains("missingFunction"), "{err}"); + let (code, v) = run_json(&["script", "--eval", "\nnope()"]); + assert_eq!(code, 1); + assert_eq!(v["error"]["line"], json!(2)); +} diff --git a/apps/effectcraft-web/src/lib.rs b/apps/effectcraft-web/src/lib.rs index 7a765bfc6..edd90e311 100644 --- a/apps/effectcraft-web/src/lib.rs +++ b/apps/effectcraft-web/src/lib.rs @@ -105,8 +105,8 @@ fn query_flag(name: &str) -> bool { web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()).is_some_and(|p| p.has(name)) } -/// A session wired for the browser: media decoding from memory, expressions, Render Queue export -/// to downloads. +/// A session wired for the browser: media decoding from memory, expressions, scripting, Render +/// Queue export to downloads. pub fn session() -> Session { let pool = Arc::new(effectcraft_media::MediaPool::new()); Session { @@ -115,6 +115,7 @@ pub fn session() -> Session { importer: Some(Arc::new(files::WebImporter { pool })), expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_host::script::runner), exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), ..Default::default() } diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index a3b164080..0b2bcb051 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -14,3 +14,6 @@ effectcraft-engine = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } image = { workspace = true } + +[dev-dependencies] +effectcraft-script = { workspace = true } diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index f175b12f9..848d3ee0b 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -54,6 +54,7 @@ fn initialize_and_list_tools() { for want in [ "list_commands", "execute_command", + "run_script", "get_project", "get_comp", "get_layer", @@ -263,3 +264,30 @@ fn bridge_rejects_non_loopback() { assert!(Backend::bridge("10.0.0.1:9877").is_err()); assert!(Backend::bridge("localhost:9877").is_ok()); } + +#[test] +fn run_script_round_trip() { + let mut sess = Session::default(); + effectcraft_script::install(&mut sess); + let mut s = McpServer::new(Backend::headless(sess)); + let r = call_json( + &mut s, + "run_script", + json!({"code": "var c = app.project.items.addComp('Scripted', 320, 180, 1, 2, 24);\nc.layers.addText('Hi');\nwriteLn('made ' + c.name);\nc.numLayers"}), + ); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"], json!(1)); + assert_eq!(r["output"], json!("made Scripted")); + // The edit is visible to the other tools. + let p = call_json(&mut s, "get_project", json!({})); + assert!(p.to_string().contains("Scripted"), "{p}"); + // Errors come back with their line. + let r = call_json(&mut s, "run_script", json!({"code": "var a = 1;\nundefinedFn();", "name": "bad.jsx"})); + assert_eq!(r["ok"], json!(false)); + assert_eq!(r["error"]["line"], json!(2)); + assert_eq!(r["error"]["file"], json!("bad.jsx")); + // Without a scripting engine the command explains itself. + let mut bare = server(); + let (c, err) = call(&mut bare, "run_script", json!({"code": "1"})); + assert!(err && c[0]["text"].as_str().unwrap().contains("not available"), "{c:?}"); +} diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 205573b65..739c5524b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -140,6 +140,15 @@ fn execute_command(b: &mut Backend, a: &Value) -> Result { json_reply(b.exec(id, params)?) } +fn run_script(b: &mut Backend, a: &Value) -> Result { + let code = get(a, "code").and_then(Value::as_str).ok_or_else(|| Error::BadArgs("missing `code` (JavaScript)".into()))?; + let mut p = json!({"code": code}); + if let Some(n) = get(a, "name") { + p["name"] = n.clone(); + } + json_reply(b.exec("script.run", p)?) +} + fn get_project(b: &mut Backend, _: &Value) -> Result { json_reply(b.exec("project.summary", json!({}))?) } @@ -386,6 +395,21 @@ static TOOLS: &[ToolDef] = &[ }, run: execute_command, }, + ToolDef { + name: "run_script", + description: "Run JavaScript with an After Effects-style scripting object model (app, app.project, items.addComp, comp.layers.addSolid/addText/addShape/addNull/addCamera/addLight, layer.property(\"ADBE Transform Group\").property(\"ADBE Position\").setValueAtTime(t, v), Effects.addProperty(\"ADBE Gaussian Blur 2\"), expressions, markers, renderQueue…). Every edit is an undoable engine command; app.beginUndoGroup/endUndoGroup make one undo step. Returns {ok, result (last expression value), output (writeLn/$.writeln/alert text), error: {message, line, column} | null}. File access is limited to the project folder unless the user allows scripts to write files.", + bridge_only: false, + schema: || { + schema( + json!({ + "code": {"type": "string", "description": "JavaScript source, e.g. `var c = app.project.items.addComp(\"Main\", 1920, 1080, 1, 5, 30); c.layers.addText(\"Hi\"); c.numLayers`."}, + "name": {"type": "string", "description": "Script name used in error messages (default `script`)."} + }), + &["code"], + ) + }, + run: run_script, + }, ToolDef { name: "get_project", description: "Project overview: file path, dirty flag, bit depth, every item (id, name, type Composition/Footage/Solid/Folder, size, duration, layer count), the active comp id, current time (s), selected layers and the undo stack.", diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2dff..5ff744c3c 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -358,9 +358,30 @@ fn collect_files(s: &mut Session, p: &Value) -> Result { // ---------------------------------------------------------------- scripts -/// Run a command script: a JSON array or JSON lines of `{"command": id, "params": {…}}` (also -/// accepts the control channel's `{"method": "engine.execute", "params": {"id", "params"}}`). +/// File ▸ Scripts ▸ Run Script File…: `.jsx` / `.js` files run as JavaScript against the After +/// Effects-style object model ([`Session::script`]); anything else is a command script: a JSON +/// array or JSON lines of `{"command": id, "params": {…}}` (also accepts the control channel's +/// `{"method": "engine.execute", "params": {"id", "params"}}`). fn run_script(s: &mut Session, p: &Value) -> Result { + if let Some(path) = str_p(p, "path") + && p.get("steps").is_none() + && is_js_script(path) + { + if path.to_ascii_lowercase().ends_with(".jsxbin") { + return Err(bad("file.runScript", "binary .jsxbin scripts are not supported: run the .jsx source")); + } + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let code = String::from_utf8_lossy(&bytes).to_string(); + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + let out = run_js(s, &code, path, false)?; + if let Some(e) = out.get("error").filter(|e| !e.is_null()) { + let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); + let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); + s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); + return Err(EngineError::Other(format!("{name}{line}: {msg}"))); + } + return Ok(out); + } let steps: Vec = match (p.get("steps"), str_p(p, "path")) { (Some(Value::Array(a)), _) => a.clone(), (_, Some(path)) => { @@ -388,7 +409,7 @@ fn run_script(s: &mut Session, p: &Value) -> Result { } _ => return Err(bad("file.runScript", format!("step {}: needs `command`", i + 1))), }; - if id == "file.runScript" { + if id == "file.runScript" || id == "script.run" { return Err(bad("file.runScript", "scripts can't run scripts")); } let r = s.execute(&id, params).map_err(|e| EngineError::Other(format!("step {} ({id}): {e}", i + 1)))?; @@ -397,6 +418,28 @@ fn run_script(s: &mut Session, p: &Value) -> Result { Ok(json!({"steps": results.len(), "results": results})) } +/// `.jsx` / `.js` (and `.jsxbin`, rejected) are JavaScript; `.json` / `.jsonl` / other files are +/// command scripts. +fn is_js_script(path: &str) -> bool { + let l = path.to_ascii_lowercase(); + l.ends_with(".jsx") || l.ends_with(".js") || l.ends_with(".jsxbin") +} + +/// Run JavaScript through the session's scripting engine. +fn run_js(s: &mut Session, code: &str, name: &str, console: bool) -> Result { + let runner = s.script.ok_or_else(|| EngineError::Other("JavaScript scripting is not available in this build".into()))?; + Ok(runner(s, &crate::ScriptRequest { code, name, console })) +} + +/// `script.run`: run JavaScript (the After Effects scripting object model: `app.project`, comps, +/// layers, properties…) and report `{ok, result, output, error}`; script errors are reported in +/// `error` (with line and column), not as a command failure. +fn script_run(s: &mut Session, p: &Value) -> Result { + let code = str_p(p, "code").ok_or_else(|| bad("script.run", "missing `code` (JavaScript)"))?.to_string(); + let name = str_p(p, "name").unwrap_or("script").to_string(); + run_js(s, &code, &name, super::b_p(p, "console").unwrap_or(false)) +} + // ---------------------------------------------------------------- footage fn interpret(s: &mut Session, p: &Value) -> Result { @@ -638,10 +681,19 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsonl/.json command script) | steps: [{command, params}]}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}]}", always, run_script ), + cmd!( + "script.run", + "Run Script", + [], + None, + "{code (JavaScript: app, app.project, CompItem, Layer, Property… like After Effects scripting), name?, console? (Script Console context)} → {ok, result, output, error: {message, line, column} | null}", + always, + script_run + ), cmd!( "file.interpretFootage", "Main...", diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f6..2e40eb639 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -25,6 +25,7 @@ pub mod paint; mod paths; mod project_items; mod prop; +mod prop_groups; pub mod puppet; mod query; mod render_queue; @@ -105,6 +106,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(text_edit::specs()); v.extend(layer_time::specs()); v.extend(prop::specs()); + v.extend(prop_groups::specs()); v.extend(project_items::specs()); v.extend(anim::specs()); v.extend(anim_tools::specs()); diff --git a/crates/engine/src/commands/project_items.rs b/crates/engine/src/commands/project_items.rs index 832f5851a..a39498961 100644 --- a/crates/engine/src/commands/project_items.rs +++ b/crates/engine/src/commands/project_items.rs @@ -140,6 +140,92 @@ fn set_comment(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// Delete project items (folders with their contents). Layers that use a deleted item and Render +/// Queue items of deleted comps go too, like After Effects' Edit ▸ Clear in the Project panel. +fn delete(s: &mut Session, p: &Value) -> Result { + let c = "project.delete"; + let ids = items_p(s, p, c)?; + let mut doomed: Vec = vec![]; + for id in s.project.items.keys() { + if ids.iter().any(|d| is_within(&s.project, *id, *d)) { + doomed.push(*id); + } + } + let n = doomed.len(); + s.edit("Delete Items", None, |proj, st| { + for id in &doomed { + proj.items.remove(id); + } + let comp_ids: Vec = proj.comps().map(|(id, _)| *id).collect(); + for cid in comp_ids { + let uses = proj.comp(cid).is_some_and(|c| c.layers.iter().any(|l| l.source.item().is_some_and(|i| doomed.contains(&i)))); + if !uses { + continue; + } + if let Some(comp) = proj.comp_mut(cid) { + let gone: Vec = + comp.layers.iter().filter(|l| l.source.item().is_some_and(|i| doomed.contains(&i))).map(|l| l.id).collect(); + comp.layers.retain(|l| !gone.contains(&l.id)); + for l in &mut comp.layers { + if l.parent.is_some_and(|x| gone.contains(&x)) { + l.parent = None; + } + if l.track_matte.is_some_and(|m| gone.contains(&m.layer)) { + l.track_matte = None; + } + } + } + } + proj.render_queue.retain(|r| !doomed.contains(&r.comp)); + st.project_selection.retain(|i| !doomed.contains(i)); + Ok(()) + })?; + s.sanitize_state(); + Ok(json!(n)) +} + +/// Duplicate project items (Edit ▸ Duplicate in the Project panel): comps get fresh layer ids and +/// a numbered name; the copies are selected. +fn duplicate(s: &mut Session, p: &Value) -> Result { + let c = "project.duplicate"; + let ids = items_p(s, p, c)?; + let mut created = vec![]; + s.edit("Duplicate", None, |proj, st| { + for id in &ids { + let it = proj.item(*id).ok_or(EngineError::Other("item vanished".into()))?.clone(); + if it.is_folder() { + return Err(bad(c, "folders can't be duplicated")); + } + let base = it.name.trim_end_matches(|ch: char| ch.is_ascii_digit()).trim_end().to_string(); + let name = (2..).map(|i| format!("{base} {i}")).find(|n| proj.find_by_name(n).is_none()).unwrap_or_default(); + let mut kind = it.kind.clone(); + if let effectcraft_project::ItemKind::Comp(comp) = &mut kind { + let comp = std::sync::Arc::make_mut(comp); + let mut map = std::collections::HashMap::new(); + for l in &mut comp.layers { + let new = effectcraft_project::LayerId(proj.alloc()); + map.insert(l.id, new); + l.id = new; + let mut next = proj.next_id; + l.props.reassign_uids(&mut next); + proj.next_id = next + 1; + } + for l in &mut comp.layers { + l.parent = l.parent.and_then(|p| map.get(&p).copied()); + if let Some(m) = &mut l.track_matte { + m.layer = map.get(&m.layer).copied().unwrap_or(m.layer); + } + } + } + let nid = proj.add_item(&name, it.label, it.parent, kind); + created.push(nid); + } + st.project_selection = created.clone(); + Ok(()) + })?; + Ok(json!({"items": created.iter().map(|i| i.0).collect::>()})) +} + pub fn specs() -> Vec { vec![ cmd!("project.select", "Select Project Items", [], None, "{items: [id|name], add?}", always, select), @@ -147,5 +233,15 @@ pub fn specs() -> Vec { cmd!("project.move", "Move to Folder", [], None, "{items?: [id|name] (default: selected), folder: id|name|null (root)}", always, move_items), cmd!("project.setLabel", "Item Label", [], None, "{items?, label: name|index}", always, set_label), cmd!("project.setComment", "Item Comment", [], None, "{items?, comment}", always, set_comment), + cmd!( + "project.delete", + "Delete Project Items", + [], + None, + "{items?: [id|name] (default: selected)} (folders with their contents; layers using the items go too)", + always, + delete + ), + cmd!("project.duplicate", "Duplicate Project Items", [], None, "{items?: [id|name] (default: selected)} → {items}", always, duplicate), ] } diff --git a/crates/engine/src/commands/prop_groups.rs b/crates/engine/src/commands/prop_groups.rs new file mode 100644 index 000000000..7f5f2a2c2 --- /dev/null +++ b/crates/engine/src/commands/prop_groups.rs @@ -0,0 +1,107 @@ +//! Generic edits of property-tree groups by uid: rename, enable, remove, duplicate and reorder an +//! effect, mask, shape item or text animator (what scripting's `PropertyGroup.name`, `enabled`, +//! `remove()`, `duplicate()` and `moveTo()` do, and what agents need without a UI selection). + +use effectcraft_project::{GroupKind, ItemId, LayerId, Node, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, has_layers, layer_mut, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// Groups that can be renamed, removed, duplicated and reordered: instances in indexed lists. +fn is_instance(g: &PropGroup) -> bool { + matches!(g.kind, GroupKind::Indexed | GroupKind::Effect { .. } | GroupKind::Mask { .. } | GroupKind::Tracker { .. }) +} + +/// `{layer, prop: uid}` → (comp, layer, group uid), checking that the group is an instance. +fn group_ref(s: &Session, p: &Value, cmd: &str, need_instance: bool) -> Result<(ItemId, LayerId, Uid)> { + let (cid, lid) = layer_p(s, p, cmd)?; + let uid = p.get("prop").and_then(Value::as_u64).ok_or_else(|| bad(cmd, "missing `prop` (group uid)"))?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let g = l.props.find_group(uid).ok_or_else(|| bad(cmd, format!("no property group @{uid}")))?; + if need_instance && !is_instance(g) { + return Err(bad(cmd, format!("`{}` is a fixed group (only effects, masks, shape items and animators)", g.name))); + } + Ok((cid, lid, uid)) +} + +fn rename(s: &mut Session, p: &Value) -> Result { + let c = "prop.renameGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name`"))?.to_string(); + if name.trim().is_empty() { + return Err(bad(c, "the name can't be empty")); + } + s.edit("Rename", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + g.name = name.clone(); + Ok(json!(name)) + }) +} + +fn set_enabled(s: &mut Session, p: &Value) -> Result { + let c = "prop.setGroupEnabled"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let v = b_p(p, "value"); + s.edit("Toggle", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + g.enabled = v.unwrap_or(!g.enabled); + Ok(json!(g.enabled)) + }) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + let c = "prop.removeGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + s.edit("Delete", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + parent.children.retain(|n| n.uid() != uid); + st.selected_props.retain(|(_, u)| *u != uid); + Ok(Value::Null) + }) +} + +fn duplicate(s: &mut Session, p: &Value) -> Result { + let c = "prop.duplicateGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + s.edit("Duplicate", None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + let i = parent.children.iter().position(|n| n.uid() == uid).ok_or_else(|| bad(c, "group vanished"))?; + let Node::Group(mut g) = parent.children[i].clone() else { return Err(bad(c, "not a group")) }; + g.reassign_uids(&mut next); + let new = g.uid; + parent.children.insert(i + 1, Node::Group(g)); + proj.next_id = next + 1; + Ok(json!({"prop": new})) + }) +} + +fn move_to(s: &mut Session, p: &Value) -> Result { + let c = "prop.moveGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let to = p.get("index").and_then(Value::as_u64).ok_or_else(|| bad(c, "missing `index` (1-based)"))? as usize; + s.edit("Reorder", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + let i = parent.children.iter().position(|n| n.uid() == uid).ok_or_else(|| bad(c, "group vanished"))?; + if to < 1 || to > parent.children.len() { + return Err(bad(c, format!("index {to} is out of range 1..{}", parent.children.len()))); + } + let n = parent.children.remove(i); + parent.children.insert(to - 1, n); + Ok(json!(to)) + }) +} + +pub fn specs() -> Vec { + vec![ + cmd!("prop.renameGroup", "Rename Property Group", [], None, "{layer?, prop: uid, name}", has_layers, rename), + cmd!("prop.setGroupEnabled", "Enable Property Group", [], None, "{layer?, prop: uid, value?}", has_layers, set_enabled), + cmd!("prop.removeGroup", "Delete Property Group", [], None, "{layer?, prop: uid}", has_layers, remove), + cmd!("prop.duplicateGroup", "Duplicate Property Group", [], None, "{layer?, prop: uid} → {prop: new uid}", has_layers, duplicate), + cmd!("prop.moveGroup", "Reorder Property Group", [], None, "{layer?, prop: uid, index (1-based among its siblings)}", has_layers, move_to), + ] +} diff --git a/crates/engine/src/commands/text_anim.rs b/crates/engine/src/commands/text_anim.rs index 73f34c5e9..51367da91 100644 --- a/crates/engine/src/commands/text_anim.rs +++ b/crates/engine/src/commands/text_anim.rs @@ -114,7 +114,8 @@ fn add_animator(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("Animator {}", anims.children.len() + 1)); let mut g = build::text_animator(&mut ids, &name, vec![]); let added: usize = kinds.iter().map(|k| add_props(&mut ids, &mut g, k, three)).sum(); - if added == 0 { + // `properties: []` adds an empty animator (scripting's addProperty("ADBE Text Animator")). + if added == 0 && !kinds.is_empty() { return Err(bad(C, "unknown animator property")); } let uid = g.uid; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488ab..063b9bdf9 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -285,8 +285,25 @@ pub struct Session { pub autosave: autosave::AutoSaveState, /// The viewer snapshot (Take Snapshot / Show Snapshot). pub snapshot: Option, + /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): + /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. + pub script: Option, } +/// A script to run (see [`Session::script`]). +#[derive(Clone, Copy, Debug)] +pub struct ScriptRequest<'a> { + pub code: &'a str, + /// File name shown in errors and `$.fileName` (`console` for the Script Console). + pub name: &'a str, + /// Run in the Script Console's persistent context (variables survive between runs). + pub console: bool, +} + +/// Runs a script against the session and reports `{ok, result, output, error: {message, line, +/// column, file} | null}`. +pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; + impl Default for Session { fn default() -> Self { Session { @@ -319,6 +336,7 @@ impl Default for Session { shortcut_table: std::sync::OnceLock::new(), autosave: autosave::AutoSaveState::default(), snapshot: None, + script: None, } } } diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 38252a884..065d1762f 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -942,6 +942,7 @@ Window Preview | window.panel {"panel":"preview"} | Cmd+3 Progress | window.unavailablePanel {"panel":"progress"} Properties | window.panel {"panel":"properties"} + Script Console | window.panel {"panel":"scriptConsole"} Smoother | window.panel {"panel":"smoother"} Tools | window.panel {"panel":"tools"} | Cmd+1 Tracker | window.panel {"panel":"tracker"} diff --git a/crates/engine/src/tests_project_items.rs b/crates/engine/src/tests_project_items.rs index 4d097aca4..4ab9ce2b2 100644 --- a/crates/engine/src/tests_project_items.rs +++ b/crates/engine/src/tests_project_items.rs @@ -39,3 +39,60 @@ fn move_into_folder_rename_label_comment_with_undo() { s.execute("edit.undo", json!({})).unwrap(); assert_eq!(s.project.item(comp).unwrap().comment, ""); } + +#[test] +fn delete_and_duplicate_items() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Inner", "width": 64, "height": 64, "duration": 1})).unwrap(); + let inner = s.active_comp_id().unwrap(); + s.execute("layer.newNull", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Outer", "width": 64, "height": 64, "duration": 1})).unwrap(); + let outer = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + s.execute("renderQueue.add", json!({"comp": inner.0})).unwrap(); + // Duplicate: a new comp with fresh layer ids. + let r = s.execute("project.duplicate", json!({"items": [inner.0]})).unwrap(); + let dup = ItemId(r["items"][0].as_u64().unwrap()); + assert_eq!(s.project.item(dup).unwrap().name, "Inner 2"); + let a = s.project.comp(inner).unwrap().layers[0].id; + let b = s.project.comp(dup).unwrap().layers[0].id; + assert_ne!(a, b); + // Delete: layers that use the item and its render items go too; undoable. + s.execute("project.delete", json!({"items": ["Inner"]})).unwrap(); + assert!(s.project.item(inner).is_none()); + assert!(s.project.comp(outer).unwrap().layers.is_empty()); + assert!(s.project.render_queue.is_empty()); + s.undo(); + assert!(s.project.item(inner).is_some()); + assert_eq!(s.project.comp(outer).unwrap().layers.len(), 1); + // Folders take their contents. + let f = ItemId(s.execute("project.newFolder", json!({"name": "F"})).unwrap()["item"].as_u64().unwrap()); + s.execute("project.move", json!({"items": [dup.0], "folder": f.0})).unwrap(); + s.execute("project.delete", json!({"items": [f.0]})).unwrap(); + assert!(s.project.item(dup).is_none()); +} + +#[test] +fn property_group_edits() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "C", "width": 64, "height": 64, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap(); + let a = s.execute("effect.apply", json!({"layer": l, "effect": "Gaussian Blur"})).unwrap()["effects"][0].as_u64().unwrap(); + let b = s.execute("effect.apply", json!({"layer": l, "effect": "Invert"})).unwrap()["effects"][0].as_u64().unwrap(); + let fx = |s: &Session| -> Vec<(u64, String, bool)> { + s.active_comp().unwrap().layers[0].props.sub("effects").unwrap().groups().map(|g| (g.uid, g.name.clone(), g.enabled)).collect() + }; + s.execute("prop.renameGroup", json!({"layer": l, "prop": a, "name": "Soft"})).unwrap(); + s.execute("prop.setGroupEnabled", json!({"layer": l, "prop": a, "value": false})).unwrap(); + s.execute("prop.moveGroup", json!({"layer": l, "prop": b, "index": 1})).unwrap(); + let d = s.execute("prop.duplicateGroup", json!({"layer": l, "prop": a})).unwrap()["prop"].as_u64().unwrap(); + let v = fx(&s); + assert_eq!(v.iter().map(|x| x.0).collect::>(), [b, a, d]); + assert_eq!(v[1].1, "Soft"); + assert!(!v[1].2); + s.execute("prop.removeGroup", json!({"layer": l, "prop": a})).unwrap(); + assert_eq!(fx(&s).len(), 2); + // Fixed groups can't be removed. + let tr = s.active_comp().unwrap().layers[0].props.sub("transform").unwrap().uid; + assert!(s.execute("prop.removeGroup", json!({"layer": l, "prop": tr})).is_err()); +} diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index b43271c70..dc7965d95 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-host" -description = "Builds a fully wired EffectCraft session: media decoding (FilmCraft codecs), importer and the expression engine" +description = "Builds a fully wired EffectCraft session: media decoding (FilmCraft codecs), importer, the expression engine and scripting" version.workspace = true edition.workspace = true license.workspace = true @@ -14,6 +14,7 @@ effectcraft-engine = { workspace = true } effectcraft-project = { workspace = true } effectcraft-media = { workspace = true } effectcraft-expr = { workspace = true } +effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } [dev-dependencies] diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 16199dfbe..5235fecaa 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -1,12 +1,14 @@ //! A fully wired [`Session`]: footage decoding through `effectcraft-media` (FilmCraft's codecs), -//! the media importer, the expression engine and Render Queue export through -//! `effectcraft-export` (FilmCraft's encoders). Frontends (desktop, CLI, MCP, web) start here. +//! the media importer, the expression engine, JavaScript scripting (`effectcraft-script`) and +//! Render Queue export through `effectcraft-export` (FilmCraft's encoders). Frontends (desktop, +//! CLI, MCP, web) start here. use std::sync::Arc; use effectcraft_engine::project::render_queue::OutputFormat; use effectcraft_engine::{ExportJob, ExportResult, Exporter, Importer, Session}; use effectcraft_project::Footage; +pub use effectcraft_script as script; struct MediaImporter; @@ -47,7 +49,7 @@ impl Exporter for FileExporter { } } -/// A new session with media, import, expressions and export enabled. +/// A new session with media, import, expressions, scripting and export enabled. pub fn session() -> Session { Session { exporter: Some(Arc::new(FileExporter::default())), @@ -55,6 +57,7 @@ pub fn session() -> Session { importer: Some(Arc::new(MediaImporter)), expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_script::runner), ..Default::default() } } @@ -188,6 +191,35 @@ mod tests { assert_eq!(value(&s, a, "transform/rotation").as_f64(), 12.0); } + /// A script builds a comp, queues it and renders it through the Render Queue. + #[test] + fn script_renders_through_the_render_queue() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-script"); + let _ = std::fs::remove_dir_all(&dir); + let out = dir.join("scripted.gif"); + let mut s = super::session(); + let code = format!( + r#" + var comp = app.project.items.addComp("Scripted", 64, 48, 1, 0.2, 10); + var sq = comp.layers.addSolid([1, 0.5, 0], "Square", 16, 16, 1); + sq.transform.position.setValueAtTime(0, [8, 24]); + sq.transform.position.setValueAtTime(0.1, [56, 24]); + var item = app.project.renderQueue.items.add(comp); + item.outputModule(1).applyTemplate("GIF"); + item.outputModule(1).file = new File({}); + app.project.renderQueue.render(); + [item.status == RQItemStatus.DONE, item.outputModule(1).file.fsName, app.project.renderQueue.numItems] + "#, + serde_json::to_string(&out.to_string_lossy()).unwrap() + ); + let r = s.execute("script.run", json!({"code": code})).unwrap(); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"][0], json!(true), "{r}"); + assert_eq!(r["result"][2], json!(1)); + let bytes = std::fs::read(&out).unwrap(); + assert!(bytes.starts_with(b"GIF89a")); + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/script/Cargo.toml b/crates/script/Cargo.toml new file mode 100644 index 000000000..ef756107c --- /dev/null +++ b/crates/script/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "effectcraft-script" +description = "EffectCraft scripting: JavaScript (boa) with an After Effects-style object model (app, project, CompItem, Layer, Property…) over the engine's undoable commands" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-engine = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-keyframe = { workspace = true } +effectcraft-time = { workspace = true } +effectcraft-render = { workspace = true } +effectcraft-path = { workspace = true } +serde_json = { workspace = true } +boa_engine = { workspace = true } + +# On the web, boa's clock and Math.random need the browser (getrandom `wasm_js`, web-time). +[target.'cfg(target_arch = "wasm32")'.dependencies] +boa_engine = { workspace = true, features = ["js"] } + +[dev-dependencies] +effectcraft-expr = { workspace = true } diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs new file mode 100644 index 000000000..a1e9e5d44 --- /dev/null +++ b/crates/script/src/lib.rs @@ -0,0 +1,64 @@ +//! # effectcraft-script (L4) +//! +//! Scripting with an After Effects-style object model, on the boa JavaScript engine (the one the +//! expressions use). Scripts written for After Effects' documented scripting API mostly run +//! unchanged: +//! +//! ```text +//! app.beginUndoGroup("Build"); +//! var comp = app.project.items.addComp("Main", 1920, 1080, 1, 10, 30); +//! var solid = comp.layers.addSolid([1, 0, 0], "Red", 1920, 1080, 1); +//! solid.property("ADBE Transform Group").property("ADBE Position").setValueAtTime(0, [0, 540]); +//! solid.Effects.addProperty("ADBE Gaussian Blur 2").property("Blurriness").setValue(20); +//! app.endUndoGroup(); +//! ``` +//! +//! * **Object model** (`prelude.js`): `app` (project, open/newProject, undo groups, +//! `executeCommand`/`findMenuCommandId`, `scheduleTask`), `Project`, `ItemCollection`, +//! `CompItem`/`FootageItem`/`FolderItem`, `LayerCollection`, `AVLayer`/`TextLayer`/ +//! `ShapeLayer`/`CameraLayer`/`LightLayer`, `PropertyGroup`/`Property` (values, keyframes, +//! eases, expressions), `MaskPropertyGroup`, `TextDocument`, `Shape`, `KeyframeEase`, +//! `MarkerValue`, `RenderQueue`/`RenderQueueItem`/`OutputModule`, `ImportOptions`, `File`/ +//! `Folder`, `$`, `alert`/`writeLn`, and the enums (`BlendingMode`, `KeyframeInterpolationType`, +//! `TrackMatteType`, `LightType`, `ParagraphJustification`, `PropertyValueType`…). +//! * **Edits are engine commands**: every mutating call runs a command through +//! [`Session::execute`](effectcraft_engine::Session::execute), so it is undoable, journaled +//! and identical to the UI's action; `app.beginUndoGroup`/`endUndoGroup` fold everything in +//! between into one undo step. +//! * **Match names** ([`matchnames`]): After Effects' documented match names for our properties +//! and effects (`ADBE Transform Group`, `ADBE Position`, `ADBE Gaussian Blur 2`…). +//! * **Security**: scripts can import/open/save projects and render, but `File` reads are limited +//! to the project's folder and writes (and the network) are off unless Preferences ▸ Scripting +//! & Expressions ▸ Allow Scripts to Write Files and Access Network is on. +//! +//! Entry points: [`install`] sets [`Session::script`](effectcraft_engine::Session::script), which +//! the `script.run` command, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`), the Script Console +//! panel, `effectcraft-cli script` and the MCP `run_script` tool use. + +#![recursion_limit = "256"] + +pub mod matchnames; +mod model; +mod runtime; + +use effectcraft_engine::{ScriptRequest, Session}; + +pub use runtime::{Outcome, ScriptError, run}; + +/// The [`effectcraft_engine::ScriptRunner`] this crate provides. +pub fn runner(s: &mut Session, req: &ScriptRequest) -> serde_json::Value { + run(s, req).to_json() +} + +/// Enable scripting on a session. +pub fn install(s: &mut Session) { + s.script = Some(runner); +} + +/// Run `code` as a one-off script named `name`. +pub fn run_code(s: &mut Session, code: &str, name: &str) -> Outcome { + run(s, &ScriptRequest { code, name, console: false }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/script/src/matchnames.rs b/crates/script/src/matchnames.rs new file mode 100644 index 000000000..623027c13 --- /dev/null +++ b/crates/script/src/matchnames.rs @@ -0,0 +1,443 @@ +//! After Effects-compatible match names for our property tree and effects, so common scripts +//! (`layer.property("ADBE Transform Group").property("ADBE Position")`, +//! `layer.Effects.addProperty("ADBE Gaussian Blur 2")`) work unchanged. +//! +//! The names are the stable identifiers After Effects documents for scripting (Scripting Guide +//! ▸ match names). Properties without a documented equivalent keep our own match id +//! (`perChar3d`, `ec.blur.bilateral`), which `property()` and `addProperty()` accept too. + +use effectcraft_project::{LayerSource, Node}; + +/// Match name of a layer (its root property group). +pub fn layer(source: &LayerSource) -> &'static str { + match source { + LayerSource::Text => "ADBE Text Layer", + LayerSource::Shape => "ADBE Vector Layer", + LayerSource::Camera => "ADBE Camera Layer", + LayerSource::Light { .. } => "ADBE Light Layer", + _ => "ADBE AV Layer", + } +} + +/// Top-level groups of a layer. +const LAYER_GROUPS: &[(&str, &str)] = &[ + ("transform", "ADBE Transform Group"), + ("masks", "ADBE Mask Parade"), + ("effects", "ADBE Effect Parade"), + ("text", "ADBE Text Properties"), + ("contents", "ADBE Root Vectors Group"), + ("materialOptions", "ADBE Material Options Group"), + ("cameraOptions", "ADBE Camera Options Group"), + ("lightOptions", "ADBE Light Options Group"), + ("audio", "ADBE Audio Group"), + ("layerStyles", "ADBE Layer Styles"), + ("motionTrackers", "ADBE MTrackers"), + ("timeRemap", "ADBE Time Remapping"), +]; + +const TRANSFORM: &[(&str, &str)] = &[ + ("anchor", "ADBE Anchor Point"), + // Cameras and lights store the Point of Interest where AVLayers keep the Anchor Point. + ("poi", "ADBE Anchor Point"), + ("position", "ADBE Position"), + ("positionX", "ADBE Position_0"), + ("positionY", "ADBE Position_1"), + ("positionZ", "ADBE Position_2"), + ("scale", "ADBE Scale"), + ("orientation", "ADBE Orientation"), + ("rotationX", "ADBE Rotate X"), + ("rotationY", "ADBE Rotate Y"), + ("rotation", "ADBE Rotate Z"), + ("opacity", "ADBE Opacity"), +]; + +const MASK: &[(&str, &str)] = + &[("path", "ADBE Mask Shape"), ("feather", "ADBE Mask Feather"), ("opacity", "ADBE Mask Opacity"), ("expansion", "ADBE Mask Offset")]; + +const TEXT: &[(&str, &str)] = &[ + ("sourceText", "ADBE Text Document"), + ("pathOptions", "ADBE Text Path Options"), + ("moreOptions", "ADBE Text More Options"), + ("animators", "ADBE Text Animators"), +]; + +const TEXT_ANIMATOR: &[(&str, &str)] = &[("selectors", "ADBE Text Selectors"), ("properties", "ADBE Text Animator Properties")]; + +const CAMERA: &[(&str, &str)] = &[ + ("zoom", "ADBE Camera Zoom"), + ("dof", "ADBE Camera Depth of Field"), + ("focusDistance", "ADBE Camera Focus Distance"), + ("aperture", "ADBE Camera Aperture"), + ("blurLevel", "ADBE Camera Blur Level"), +]; + +const LIGHT: &[(&str, &str)] = &[ + ("intensity", "ADBE Light Intensity"), + ("color", "ADBE Light Color"), + ("coneAngle", "ADBE Light Cone Angle"), + ("coneFeather", "ADBE Light Cone Feather 2"), + ("falloff", "ADBE Light Falloff Type"), + ("radius", "ADBE Light Falloff Start"), + ("falloffDistance", "ADBE Light Falloff Distance"), + ("castsShadows", "ADBE Light Casts Shadows"), + ("shadowDarkness", "ADBE Light Shadow Darkness"), + ("shadowDiffusion", "ADBE Light Shadow Diffusion"), +]; + +const MATERIAL: &[(&str, &str)] = &[ + ("castsShadows", "ADBE Casts Shadows"), + ("lightTransmission", "ADBE Light Transmission"), + ("acceptsShadows", "ADBE Accepts Shadows"), + ("acceptsLights", "ADBE Accepts Lights"), + ("ambient", "ADBE Ambient Coefficient"), + ("diffuse", "ADBE Diffuse Coefficient"), + ("specularIntensity", "ADBE Specular Coefficient"), + ("specularShininess", "ADBE Shininess Coefficient"), + ("metal", "ADBE Metal Coefficient"), +]; + +/// Shape items (Contents ▸ Add): our match id → AE match name. +pub const SHAPE_ITEMS: &[(&str, &str)] = &[ + ("group", "ADBE Vector Group"), + ("rect", "ADBE Vector Shape - Rect"), + ("ellipse", "ADBE Vector Shape - Ellipse"), + ("star", "ADBE Vector Shape - Star"), + ("path", "ADBE Vector Shape - Group"), + ("fill", "ADBE Vector Graphic - Fill"), + ("stroke", "ADBE Vector Graphic - Stroke"), + ("gfill", "ADBE Vector Graphic - G-Fill"), + ("gstroke", "ADBE Vector Graphic - G-Stroke"), + ("merge", "ADBE Vector Filter - Merge"), + ("offset", "ADBE Vector Filter - Offset"), + ("pucker", "ADBE Vector Filter - PB"), + ("repeater", "ADBE Vector Filter - Repeater"), + ("round", "ADBE Vector Filter - RC"), + ("trim", "ADBE Vector Filter - Trim"), + ("twist", "ADBE Vector Filter - Twist"), + ("wiggle", "ADBE Vector Filter - Roughen"), + ("zigzag", "ADBE Vector Filter - Zigzag"), +]; + +/// Properties inside shape items: (item match id, property match id, AE match name). +const SHAPE_PROPS: &[(&str, &str, &str)] = &[ + ("group", "blend", "ADBE Vector Blend Mode"), + ("group", "contents", "ADBE Vectors Group"), + ("group", "transform", "ADBE Vector Transform Group"), + ("transform", "anchor", "ADBE Vector Anchor"), + ("transform", "position", "ADBE Vector Position"), + ("transform", "scale", "ADBE Vector Scale"), + ("transform", "skew", "ADBE Vector Skew"), + ("transform", "skewAxis", "ADBE Vector Skew Axis"), + ("transform", "rotation", "ADBE Vector Rotation"), + ("transform", "opacity", "ADBE Vector Group Opacity"), + ("rect", "direction", "ADBE Vector Shape Direction"), + ("rect", "size", "ADBE Vector Rect Size"), + ("rect", "position", "ADBE Vector Rect Position"), + ("rect", "roundness", "ADBE Vector Rect Roundness"), + ("ellipse", "direction", "ADBE Vector Shape Direction"), + ("ellipse", "size", "ADBE Vector Ellipse Size"), + ("ellipse", "position", "ADBE Vector Ellipse Position"), + ("star", "direction", "ADBE Vector Shape Direction"), + ("star", "type", "ADBE Vector Star Type"), + ("star", "points", "ADBE Vector Star Points"), + ("star", "position", "ADBE Vector Star Position"), + ("star", "rotation", "ADBE Vector Star Rotation"), + ("star", "innerRadius", "ADBE Vector Star Inner Radius"), + ("star", "outerRadius", "ADBE Vector Star Outer Radius"), + ("star", "innerRoundness", "ADBE Vector Star Inner Roundess"), + ("star", "outerRoundness", "ADBE Vector Star Outer Roundess"), + ("path", "direction", "ADBE Vector Shape Direction"), + ("path", "path", "ADBE Vector Shape"), + ("fill", "blend", "ADBE Vector Blend Mode"), + ("fill", "composite", "ADBE Vector Composite Order"), + ("fill", "rule", "ADBE Vector Fill Rule"), + ("fill", "color", "ADBE Vector Fill Color"), + ("fill", "opacity", "ADBE Vector Fill Opacity"), + ("stroke", "blend", "ADBE Vector Blend Mode"), + ("stroke", "composite", "ADBE Vector Composite Order"), + ("stroke", "color", "ADBE Vector Stroke Color"), + ("stroke", "opacity", "ADBE Vector Stroke Opacity"), + ("stroke", "width", "ADBE Vector Stroke Width"), + ("stroke", "cap", "ADBE Vector Stroke Line Cap"), + ("stroke", "join", "ADBE Vector Stroke Line Join"), + ("stroke", "miter", "ADBE Vector Stroke Miter Limit"), + ("stroke", "dashes", "ADBE Vector Stroke Dashes"), + ("trim", "start", "ADBE Vector Trim Start"), + ("trim", "end", "ADBE Vector Trim End"), + ("trim", "offset", "ADBE Vector Trim Offset"), + ("trim", "mode", "ADBE Vector Trim Type"), + ("repeater", "copies", "ADBE Vector Repeater Copies"), + ("repeater", "offset", "ADBE Vector Repeater Offset"), + ("repeater", "composite", "ADBE Vector Repeater Order"), + ("repeater", "transform", "ADBE Vector Repeater Transform"), + ("round", "radius", "ADBE Vector RoundCorner Radius"), + ("pucker", "amount", "ADBE Vector PuckerBloat Amount"), + ("twist", "angle", "ADBE Vector Twist Angle"), + ("twist", "center", "ADBE Vector Twist Center"), + ("merge", "mode", "ADBE Vector Merge Type"), +]; + +/// Effects: our effect id → AE match name (the documented ones; the rest keep our id). +pub const EFFECTS: &[(&str, &str)] = &[ + ("ec.blur.gaussian", "ADBE Gaussian Blur 2"), + ("ec.blur.fastbox", "ADBE Box Blur2"), + ("ec.blur.directional", "ADBE Motion Blur"), + ("ec.blur.radial", "ADBE Radial Blur"), + ("ec.blur.cameralens", "ADBE Camera Lens Blur"), + ("ec.blur.channel", "ADBE Channel Blur"), + ("ec.blur.compound", "ADBE Compound Blur"), + ("ec.blur.sharpen", "ADBE Sharpen"), + ("ec.blur.smart", "ADBE Smart Blur"), + ("ec.blur.unsharp", "ADBE Unsharp Mask2"), + ("ec.channel.invert", "ADBE Invert"), + ("ec.channel.setmatte", "ADBE Set Matte3"), + ("ec.channel.setchannels", "ADBE Set Channels"), + ("ec.channel.shiftchannels", "ADBE Shift Channels"), + ("ec.channel.minimax", "ADBE Minimax"), + ("ec.channel.blend", "ADBE Blend"), + ("ec.channel.arithmetic", "ADBE Arithmetic"), + ("ec.channel.calculations", "ADBE Calculations"), + ("ec.channel.solidcomposite", "ADBE Solid Composite"), + ("ec.color.brightnesscontrast", "ADBE Brightness & Contrast 2"), + ("ec.color.curves", "ADBE CurvesCustom"), + ("ec.color.levels", "ADBE Easy Levels2"), + ("ec.color.levelsic", "ADBE Pro Levels2"), + ("ec.color.huesaturation", "ADBE HUE SATURATION"), + ("ec.color.exposure", "ADBE Exposure2"), + ("ec.color.tint", "ADBE Tint"), + ("ec.color.tritone", "ADBE Tritone"), + ("ec.color.blackwhite", "ADBE Black&White"), + ("ec.color.channelmixer", "ADBE CHANNEL MIXER"), + ("ec.color.colorbalance", "ADBE Color Balance 2"), + ("ec.color.colorbalancehls", "ADBE Color Balance (HLS)"), + ("ec.color.changecolor", "ADBE Change Color"), + ("ec.color.changetocolor", "ADBE Change To Color"), + ("ec.color.colorama", "APC Colorama"), + ("ec.color.leavecolor", "ADBE Leave Color"), + ("ec.color.photofilter", "ADBE Photo Filter"), + ("ec.color.vibrance", "ADBE Vibrance"), + ("ec.color.lumetri", "ADBE Lumetri"), + ("ec.color.shadowhighlight", "ADBE Shadow/Highlight"), + ("ec.color.selectivecolor", "ADBE Selective Color"), + ("ec.color.gammapedestalgain", "ADBE Gamma/Pedestal/Gain"), + ("ec.color.equalize", "ADBE Equalize"), + ("ec.color.autocontrast", "ADBE AutoContrast"), + ("ec.color.autolevels", "ADBE AutoLevels"), + ("ec.color.autocolor", "ADBE AutoColor"), + ("ec.control.slider", "ADBE Slider Control"), + ("ec.control.angle", "ADBE Angle Control"), + ("ec.control.checkbox", "ADBE Checkbox Control"), + ("ec.control.color", "ADBE Color Control"), + ("ec.control.point", "ADBE Point Control"), + ("ec.control.point3d", "ADBE Point3D Control"), + ("ec.control.layer", "ADBE Layer Control"), + ("ec.control.dropdown", "ADBE Dropdown Control"), + ("ec.distort.transform", "ADBE Geometry2"), + ("ec.distort.cornerpin", "ADBE Corner Pin"), + ("ec.distort.bulge", "ADBE Bulge"), + ("ec.distort.mirror", "ADBE Mirror"), + ("ec.distort.offset", "ADBE Offset"), + ("ec.distort.ripple", "ADBE Ripple"), + ("ec.distort.spherize", "ADBE Spherize"), + ("ec.distort.twirl", "ADBE Twirl"), + ("ec.distort.wavewarp", "ADBE Wave Warp"), + ("ec.distort.turbulentdisplace", "ADBE Turbulent Displace"), + ("ec.distort.displacementmap", "ADBE Displacement Map"), + ("ec.distort.meshwarp", "ADBE MESH WARP"), + ("ec.distort.polar", "ADBE Polar Coordinates"), + ("ec.distort.opticscompensation", "ADBE Optics Compensation"), + ("ec.distort.magnify", "ADBE Magnify"), + ("ec.distort.warpstabilizer", "ADBE SubspaceStabilizer"), + ("ec.distort.puppet", "ADBE FreePin3"), + ("ec.generate.fill", "ADBE Fill"), + ("ec.generate.gradientramp", "ADBE Ramp"), + ("ec.generate.fourcolor", "ADBE 4ColorGradient"), + ("ec.generate.stroke", "ADBE Stroke"), + ("ec.generate.grid", "ADBE Grid"), + ("ec.generate.checkerboard", "ADBE Checkerboard"), + ("ec.generate.circle", "ADBE Circle"), + ("ec.generate.ellipse", "ADBE Ellipse"), + ("ec.generate.cellpattern", "ADBE Cell Pattern"), + ("ec.generate.lensflare", "ADBE Lens Flare"), + ("ec.generate.vegas", "ADBE Vegas"), + ("ec.generate.writeon", "ADBE Write On"), + ("ec.generate.audiospectrum", "ADBE AudSpect"), + ("ec.generate.audiowaveform", "ADBE AudWave"), + ("ec.generate.beam", "ADBE Laser"), + ("ec.generate.radiowaves", "ADBE Radio Waves"), + ("ec.generate.scribble", "ADBE Scribble Fill"), + ("ec.generate.paintbucket", "ADBE Paint Bucket"), + ("ec.key.colorkey", "ADBE Color Key"), + ("ec.key.luma", "ADBE Luma Key"), + ("ec.key.linearcolor", "ADBE Linear Color Key2"), + ("ec.key.colorrange", "ADBE Color Range"), + ("ec.key.colordifference", "ADBE Color Difference Key"), + ("ec.key.extract", "ADBE Extract"), + ("ec.key.spill", "ADBE Spill2"), + ("ec.matte.simplechoker", "ADBE Simple Choker"), + ("ec.matte.mattechoker", "ADBE Matte Choker"), + ("ec.noise.noise", "ADBE Noise"), + ("ec.noise.fractal", "ADBE Fractal Noise"), + ("ec.noise.median", "ADBE Median"), + ("ec.noise.addgrain", "VISINF Grain Implant"), + ("ec.noise.dustscratches", "ADBE Dust & Scratches"), + ("ec.noise.turbulent", "ADBE Turbulent Noise"), + ("ec.perspective.dropshadow", "ADBE Drop Shadow"), + ("ec.perspective.radialshadow", "ADBE Radial Shadow"), + ("ec.perspective.bevelalpha", "ADBE Bevel Alpha"), + ("ec.perspective.beveledges", "ADBE Bevel Edges"), + ("ec.perspective.3dglasses", "ADBE 3D Glasses2"), + ("ec.stylize.glow", "ADBE Glo2"), + ("ec.stylize.mosaic", "ADBE Mosaic"), + ("ec.stylize.posterize", "ADBE Posterize"), + ("ec.stylize.threshold", "ADBE Threshold2"), + ("ec.stylize.findedges", "ADBE Find Edges"), + ("ec.stylize.emboss", "ADBE Emboss"), + ("ec.stylize.coloremboss", "ADBE Color Emboss"), + ("ec.stylize.roughenedges", "ADBE Roughen Edges"), + ("ec.stylize.scatter", "ADBE Scatter"), + ("ec.stylize.strobe", "ADBE Strobe"), + ("ec.stylize.motiontile", "ADBE Tile"), + ("ec.stylize.texturize", "ADBE Texturize"), + ("ec.stylize.cartoon", "ADBE Cartoonify"), + ("ec.stylize.brushstrokes", "ADBE Brush Strokes"), + ("ec.text.numbers", "ADBE Numbers2"), + ("ec.text.timecode", "ADBE Timecode"), + ("ec.time.echo", "ADBE Echo"), + ("ec.time.posterizetime", "ADBE Posterize Time"), + ("ec.time.timedisplacement", "ADBE Time Displacement"), + ("ec.time.timedifference", "ADBE Time Difference"), + ("ec.time.pixelmotionblur", "ADBE Pixel Motion Blur"), + ("ec.time.timewarp", "ADBE Timewarp"), + ("ec.transition.linearwipe", "ADBE Linear Wipe"), + ("ec.transition.radialwipe", "ADBE Radial Wipe"), + ("ec.transition.venetian", "ADBE Venetian Blinds"), + ("ec.transition.blockdissolve", "ADBE Block Dissolve"), + ("ec.transition.gradientwipe", "ADBE Gradient Wipe"), + ("ec.transition.iriswipe", "ADBE Iris Wipe"), + ("ec.utility.applylut", "ADBE Apply Color LUT2"), + ("ec.utility.cineon", "ADBE Cineon Converter2"), + ("ec.utility.growbounds", "ADBE Grow Bounds"), +]; + +fn find<'a>(t: &'a [(&str, &'a str)], k: &str) -> Option<&'a str> { + t.iter().find(|(a, _)| *a == k).map(|(_, b)| *b) +} + +/// AE match name of an effect id (falls back to the id). +pub fn effect(id: &str) -> String { + find(EFFECTS, id).map(str::to_string).unwrap_or_else(|| id.to_string()) +} + +/// Our effect id for an AE effect match name. +pub fn effect_from_match(m: &str) -> Option<&'static str> { + EFFECTS.iter().find(|(_, b)| b.eq_ignore_ascii_case(m)).map(|(a, _)| *a) +} + +/// Our shape item kind (`layer.addShapeItem`) for an AE match name, our match id or a display +/// name (`Rectangle`, `Fill`, `Trim Paths`). +pub fn shape_kind(name: &str) -> Option<&'static str> { + if let Some((k, _)) = SHAPE_ITEMS.iter().find(|(k, m)| m.eq_ignore_ascii_case(name) || k.eq_ignore_ascii_case(name)) { + return Some(k); + } + let n: String = name.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase(); + Some(match n.as_str() { + "rectangle" | "rectanglepath" => "rect", + "ellipsepath" => "ellipse", + "polystar" | "polystarpath" => "star", + "polygon" => "polygon", + "group" | "vectorgroup" => "group", + "fill" => "fill", + "stroke" => "stroke", + "gradientfill" => "gfill", + "gradientstroke" => "gstroke", + "trimpaths" | "trim" => "trim", + "repeater" => "repeater", + "roundcorners" => "round", + "offsetpaths" => "offset", + "puckerbloat" | "puckerandbloat" => "pucker", + "twist" => "twist", + "zigzag" => "zigzag", + "wigglepaths" => "wiggle", + "mergepaths" => "merge", + _ => return None, + }) +} + +/// Match name of the node at the end of `chain` (layer root's children → … → node). `effect` +/// params are numbered `-0001…` in parameter order, like After Effects. +pub fn node(chain: &[&Node]) -> String { + let Some(last) = chain.last() else { return String::new() }; + let id = last.match_id(); + let parent = chain.len().checked_sub(2).map(|i| chain[i]); + let grand = chain.len().checked_sub(3).map(|i| chain[i]); + let top = chain.first().map(|n| n.match_id()).unwrap_or(""); + let own = || id.to_string(); + if chain.len() == 1 { + return find(LAYER_GROUPS, id).map(str::to_string).unwrap_or_else(own); + } + let pid = parent.map(|p| p.match_id()).unwrap_or(""); + match top { + "transform" if chain.len() == 2 => find(TRANSFORM, id).map(str::to_string).unwrap_or_else(own), + "masks" if chain.len() == 2 => "ADBE Mask Atom".into(), + "masks" if chain.len() == 3 => find(MASK, id).map(str::to_string).unwrap_or_else(own), + "effects" if chain.len() == 2 => effect(id), + "effects" if chain.len() == 3 => { + let fx = parent.and_then(Node::as_group); + match fx { + Some(g) => { + let i = g.children.iter().position(|c| c.uid() == last.uid()).unwrap_or(0) + 1; + format!("{}-{i:04}", effect(&g.match_id)) + } + None => own(), + } + } + "text" if chain.len() == 2 => find(TEXT, id).map(str::to_string).unwrap_or_else(own), + "text" if pid == "animators" => "ADBE Text Animator".into(), + "text" if grand.map(|g| g.match_id()) == Some("animators") => find(TEXT_ANIMATOR, id).map(str::to_string).unwrap_or_else(own), + "cameraOptions" => find(CAMERA, id).map(str::to_string).unwrap_or_else(own), + "lightOptions" => find(LIGHT, id).map(str::to_string).unwrap_or_else(own), + "materialOptions" => find(MATERIAL, id).map(str::to_string).unwrap_or_else(own), + "audio" if id == "levels" => "ADBE Audio Levels".into(), + "contents" => { + // Items sit in a `contents` list; their properties under the item. + if pid == "contents" { + if id == "star" || id == "polygon" { + return "ADBE Vector Shape - Star".into(); + } + return find(SHAPE_ITEMS, id).map(str::to_string).unwrap_or_else(own); + } + if let Some((_, _, m)) = SHAPE_PROPS.iter().find(|(item, prop, _)| *item == pid && *prop == id) { + return m.to_string(); + } + own() + } + _ => own(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effect_names_round_trip() { + for (id, m) in EFFECTS { + assert_eq!(effect(id), *m); + assert_eq!(effect_from_match(m), Some(*id)); + } + assert_eq!(effect("ec.blur.ccvector"), "ec.blur.ccvector"); + let mut ids: Vec<&str> = EFFECTS.iter().map(|(a, _)| *a).collect(); + ids.sort(); + ids.dedup(); + assert_eq!(ids.len(), EFFECTS.len(), "duplicate effect ids"); + } + + #[test] + fn shape_kinds() { + assert_eq!(shape_kind("ADBE Vector Shape - Rect"), Some("rect")); + assert_eq!(shape_kind("Trim Paths"), Some("trim")); + assert_eq!(shape_kind("ADBE Vector Graphic - Fill"), Some("fill")); + assert_eq!(shape_kind("nope"), None); + } +} diff --git a/crates/script/src/model.rs b/crates/script/src/model.rs new file mode 100644 index 000000000..d1dd47683 --- /dev/null +++ b/crates/script/src/model.rs @@ -0,0 +1,679 @@ +//! Read-only queries behind the object model (`__query(kind, json)`): project, items, layers, +//! property nodes, values and keyframes as JSON shaped for the prelude. Every edit goes through +//! engine commands instead (`__exec`), so it is undoable and journaled. + +use effectcraft_engine::Session; +use effectcraft_engine::color::Label; +use effectcraft_keyframe::{Interp, TextDoc, Value as KV}; +use effectcraft_project::{Comp, GroupKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Node, ParamUi, PropGroup}; +use effectcraft_render::EvalCtx; +use effectcraft_time::Tick; +use serde_json::{Value as J, json}; + +use crate::matchnames; + +type R = Result; + +fn u(a: &J, k: &str) -> Option { + a.get(k).and_then(J::as_u64) +} +fn f(a: &J, k: &str) -> Option { + a.get(k).and_then(J::as_f64) +} + +fn comp_of<'a>(s: &'a Session, a: &J) -> Result<(ItemId, &'a Comp), String> { + let id = ItemId(u(a, "comp").ok_or("missing comp")?); + s.project.comp(id).map(|c| (id, c)).ok_or_else(|| "the composition no longer exists".to_string()) +} + +fn layer_of<'a>(s: &'a Session, a: &J) -> Result<(ItemId, &'a Comp, &'a Layer), String> { + let (cid, c) = comp_of(s, a)?; + let l = c.layer(LayerId(u(a, "layer").ok_or("missing layer")?)).ok_or("the layer no longer exists (it was deleted)")?; + Ok((cid, c, l)) +} + +pub fn query(s: &mut Session, kind: &str, a: &J) -> R { + match kind { + "project" => Ok(project(s)), + "item" => item(s, a), + "layer" => layer(s, a), + "node" => node(s, a), + "child" => child(s, a), + "value" => value(s, a), + "keys" => keys(s, a), + "textdoc" => textdoc(s, a), + "markers" => markers(s, a), + "sourceRect" => source_rect(s, a), + "resolveAdd" => resolve_add(s, a), + "menuCommandId" => Ok(menu_command_id(a.get("name").and_then(J::as_str).unwrap_or(""))), + "commandAt" => { + let i = u(a, "index").unwrap_or(0) as usize; + Ok(effectcraft_engine::command_specs().get(i.wrapping_sub(1)).map(|c| json!(c.id)).unwrap_or(J::Null)) + } + "exprError" => { + let code = a.get("code").and_then(J::as_str).unwrap_or(""); + Ok(match s.expr_check { + Some(check) => json!(check(code).err().unwrap_or_default()), + None => json!(""), + }) + } + "labels" => Ok(json!(Label::ALL.iter().map(|l| l.name()).collect::>())), + "effects" => Ok(json!( + effectcraft_engine::effects::registry() + .iter() + .map(|e| json!({"id": e.id, "name": e.name, "matchName": matchnames::effect(e.id), "category": e.category})) + .collect::>() + )), + _ => Err(format!("unknown query `{kind}`")), + } +} + +fn project(s: &Session) -> J { + let p = &s.project; + let items: Vec = p.items.keys().map(|k| k.0).collect(); + let active = s.active_comp_id().map(|c| c.0).or_else(|| (s.state.project_selection.len() == 1).then(|| s.state.project_selection[0].0)); + json!({ + "items": items, + "active": active, + "selection": s.state.project_selection.iter().map(|i| i.0).collect::>(), + "path": s.path, + "dirty": s.is_dirty(), + "bitsPerChannel": p.settings.bit_depth.bits(), + "linearBlending": p.settings.blend_linear, + "linearizeWorkingSpace": p.settings.linearize, + "workingSpace": p.settings.working_space.map(|w| format!("{w:?}")).unwrap_or_default(), + "timeDisplayFrames": p.settings.time_display == effectcraft_project::TimeDisplayStyle::Frames, + "renderQueue": p.render_queue.len(), + "revision": s.revision, + }) +} + +fn label_index(l: Label) -> usize { + Label::ALL.iter().position(|x| *x == l).unwrap_or(0) +} + +fn item(s: &Session, a: &J) -> R { + let id = ItemId(u(a, "id").ok_or("missing id")?); + let it = s.project.item(id).ok_or("the item no longer exists (it was deleted)")?; + let mut o = json!({ + "id": id.0, + "name": it.name, + "typeName": it.type_name(), + "parent": it.parent.map(|p| p.0).unwrap_or(0), + "comment": it.comment, + "label": label_index(it.label), + "selected": s.state.project_selection.contains(&id), + }); + let used_in: Vec = s.project.comps().filter(|(_, c)| c.layers.iter().any(|l| l.source.item() == Some(id))).map(|(cid, _)| cid.0).collect(); + o["usedIn"] = json!(used_in); + match &it.kind { + ItemKind::Folder => { + o["kind"] = json!("folder"); + o["children"] = json!(s.project.items.values().filter(|c| c.parent == Some(id)).map(|c| c.id.0).collect::>()); + } + ItemKind::Comp(c) => { + let cid = id; + o["kind"] = json!("comp"); + let sel: Vec = s.state.selected_layers.iter().filter(|l| c.layer(**l).is_some()).map(|l| l.0).collect(); + let fps = c.frame_rate.as_f64(); + if let Some(m) = o.as_object_mut() { + m.extend( + json!({ + "width": c.width, "height": c.height, "pixelAspect": c.pixel_aspect, + "duration": c.duration.seconds(), "frameRate": fps, "frameDuration": c.frame_duration().seconds(), + "layers": c.layers.iter().map(|l| l.id.0).collect::>(), + "selectedLayers": sel, + "selectedProperties": s.state.selected_props.iter().filter(|(l, _)| c.layer(*l).is_some()).map(|(l, p)| json!([l.0, p])).collect::>(), + "time": s.time_of(cid).seconds(), + "workAreaStart": c.work_area.0.seconds(), "workAreaDuration": (c.work_area.1 - c.work_area.0).seconds(), + "bgColor": c.background, "displayStartTime": c.display_start.seconds(), + "hideShyLayers": c.hide_shy, "motionBlur": c.enable_motion_blur, "frameBlending": c.enable_frame_blending, + "draft3d": c.draft_3d, "shutterAngle": c.shutter_angle, "shutterPhase": c.shutter_phase, + "motionBlurSamplesPerFrame": c.motion_blur_samples, + "renderer": match c.renderer { effectcraft_project::Renderer::Classic3D => "ADBE Classic 3D", effectcraft_project::Renderer::Advanced3D => "ADBE Advanced 3d" }, + "open": s.state.open_comps.contains(&cid), "active": s.active_comp_id() == Some(cid), + "hasVideo": true, "hasAudio": c.layers.iter().any(|l| l.switches.audio && matches!(l.source, LayerSource::Footage { .. } | LayerSource::Comp { .. })), + "posterTime": c.poster_time.seconds(), + }) + .as_object() + .cloned() + .unwrap_or_default(), + ) + } + } + ItemKind::Footage(fo) => { + o["kind"] = json!("footage"); + o["file"] = json!(fo.path); + o["width"] = json!(fo.width); + o["height"] = json!(fo.height); + o["pixelAspect"] = json!(fo.pixel_aspect); + o["duration"] = json!(if fo.kind == effectcraft_project::FootageKind::Still { 0.0 } else { fo.duration.seconds() }); + o["frameRate"] = json!(if fo.kind == effectcraft_project::FootageKind::Still { 0.0 } else { fo.frame_rate.as_f64() }); + o["hasVideo"] = json!(fo.has_video); + o["hasAudio"] = json!(fo.has_audio); + o["footageMissing"] = json!(fo.missing); + o["isStill"] = json!(fo.kind == effectcraft_project::FootageKind::Still); + o["loop"] = json!(fo.loop_count); + } + ItemKind::Solid(so) => { + o["kind"] = json!("solid"); + o["width"] = json!(so.width); + o["height"] = json!(so.height); + o["pixelAspect"] = json!(so.pixel_aspect); + o["color"] = json!(so.color); + o["duration"] = json!(0.0); + o["frameRate"] = json!(0.0); + o["hasVideo"] = json!(true); + o["hasAudio"] = json!(false); + o["isStill"] = json!(true); + } + } + Ok(o) +} + +fn layer(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let kind = match &l.source { + LayerSource::Text => "text", + LayerSource::Shape => "shape", + LayerSource::Camera => "camera", + LayerSource::Light { .. } => "light", + _ => "av", + }; + let (w, h) = effectcraft_render::source_size(&s.project, l); + let t = s.time_of(cid); + Ok(json!({ + "id": l.id.0, + "comp": cid.0, + "kind": kind, + "name": l.name, + "index": c.index_of(l.id).unwrap_or(0), + "matchName": matchnames::layer(&l.source), + "root": l.props.uid, + "source": l.source.item().map(|i| i.0), + "nullLayer": matches!(l.source, LayerSource::Null), + "lightType": match &l.source { LayerSource::Light { kind } => kind.label(), _ => "" }, + "enabled": l.switches.video, + "audioEnabled": l.switches.audio, + "solo": l.switches.solo, + "locked": l.switches.locked, + "shy": l.switches.shy, + "collapse": l.switches.collapse, + "quality": format!("{:?}", l.switches.quality), + "samplingQuality": format!("{:?}", l.switches.sampling), + "effectsActive": l.switches.effects, + "frameBlending": format!("{:?}", l.switches.frame_blend), + "motionBlur": l.switches.motion_blur, + "adjustmentLayer": l.switches.adjustment, + "threeDLayer": l.switches.three_d, + "guideLayer": l.switches.guide, + "preserveTransparency": l.preserve_transparency, + "inPoint": l.in_point.seconds(), + "outPoint": l.out_point.seconds(), + "startTime": l.start_time.seconds(), + "stretch": l.stretch, + "time": l.layer_time(t).seconds(), + "active": l.switches.video && l.is_active_at(t), + "parent": l.parent.map(|p| p.0), + "blendingMode": l.blend_mode.label(), + "trackMatte": l.track_matte.map(|m| json!({"layer": m.layer.0, "kind": format!("{:?}", m.kind)})), + "isTrackMatte": c.layers.iter().any(|x| x.track_matte.is_some_and(|m| m.layer == l.id)), + "label": label_index(l.label), + "comment": l.comment, + "selected": s.state.selected_layers.contains(&l.id), + "autoOrient": format!("{:?}", l.auto_orient), + "width": w, "height": h, + "hasVideo": l.source.is_av(), + "hasAudio": matches!(l.source, LayerSource::Footage { .. } | LayerSource::Comp { .. }) && l.props.sub("audio").is_some(), + "timeRemapEnabled": l.props.prop("timeRemap").is_some(), + })) +} + +/// Node at the end of a uid chain below the layer root, and the chain itself. +fn chain(l: &Layer, uid: u64) -> Option> { + l.props.node_chain(uid) +} + +/// PropertyValueType of a property. +fn value_type(p: &effectcraft_project::Property) -> &'static str { + if matches!(p.ui, ParamUi::Mask) { + return "MASK_INDEX"; + } + match &p.value { + KV::Scalar(_) | KV::Bool(_) | KV::Enum(_) => "OneD", + KV::Vec2(_) => { + if p.spatial { + "TwoD_SPATIAL" + } else { + "TwoD" + } + } + KV::Vec3(_) => { + if p.spatial { + "ThreeD_SPATIAL" + } else { + "ThreeD" + } + } + KV::Color(_) => "COLOR", + KV::Path(_) => "SHAPE", + KV::Text(_) => "TEXT_DOCUMENT", + KV::Layer(_) => "LAYER_INDEX", + KV::Gradient(_) | KV::Str(_) => "CUSTOM_VALUE", + } +} + +/// Groups whose children are instances (effects, masks, shape items, animators). +fn indexed_parade(g: &PropGroup) -> bool { + matches!(g.match_id.as_str(), "effects" | "masks" | "contents" | "animators" | "selectors" | "layerStyles" | "motionTrackers") +} + +fn node_json(c: &Comp, l: &Layer, uid: u64) -> R { + if uid == 0 || uid == l.props.uid { + let n = l.props.children.len(); + return Ok(json!({ + "uid": l.props.uid, "kind": "group", "name": l.name, "matchName": matchnames::layer(&l.source), "matchId": "layer", + "index": c.index_of(l.id).unwrap_or(0), "depth": 0, "parent": null, "isLayer": true, "numProperties": n, + "children": l.props.children.iter().map(Node::uid).collect::>(), "propertyType": "NAMED_GROUP", + "enabled": l.switches.video, "canSetEnabled": true, + })); + } + let ch = chain(l, uid).ok_or("the property no longer exists (it was deleted)")?; + let node = *ch.last().ok_or("bad property")?; + let parent_group: &PropGroup = if ch.len() >= 2 { ch[ch.len() - 2].as_group().ok_or("bad tree")? } else { &l.props }; + let index = parent_group.children.iter().position(|x| x.uid() == uid).map(|i| i + 1).unwrap_or(0); + let parent_uid = if ch.len() >= 2 { ch[ch.len() - 2].uid() } else { l.props.uid }; + let match_name = matchnames::node(&ch); + let instance_parent = indexed_parade(parent_group) && ch.len() >= 2 || (ch.len() == 2 && ch[0].match_id() == "contents"); + let mut o = json!({ + "uid": uid, "name": node.name(), "matchName": match_name, "matchId": node.match_id(), "index": index, + "depth": ch.len(), "parent": parent_uid, "isLayer": false, + "isEffect": matches!(node.as_group().map(|g| &g.kind), Some(GroupKind::Effect { .. })), + "isMask": matches!(node.as_group().map(|g| &g.kind), Some(GroupKind::Mask { .. })), + }); + match node { + Node::Group(g) => { + let instance = matches!(g.kind, GroupKind::Indexed | GroupKind::Effect { .. } | GroupKind::Mask { .. } | GroupKind::Tracker { .. }); + o["kind"] = json!("group"); + o["numProperties"] = json!(g.children.len()); + o["children"] = json!(g.children.iter().map(Node::uid).collect::>()); + o["propertyType"] = json!(if indexed_parade(g) { "INDEXED_GROUP" } else { "NAMED_GROUP" }); + o["enabled"] = json!(g.enabled); + o["canSetEnabled"] = json!(instance); + o["instance"] = json!(instance || instance_parent); + o["effectId"] = json!(match &g.kind { + GroupKind::Effect { effect } => Some(effect.clone()), + _ => None, + }); + if let GroupKind::Mask { mode, inverted, color, locked, .. } = &g.kind { + o["maskMode"] = json!(mode.label()); + o["inverted"] = json!(inverted); + o["color"] = json!([color[0] as f64 / 255.0, color[1] as f64 / 255.0, color[2] as f64 / 255.0]); + o["locked"] = json!(locked); + } + } + Node::Prop(p) => { + o["kind"] = json!("prop"); + o["propertyType"] = json!("PROPERTY"); + o["valueType"] = json!(value_type(p)); + o["type"] = json!(p.value.kind_name()); + o["dims"] = json!(p.value.dims()); + o["spatial"] = json!(p.spatial); + o["numKeys"] = json!(p.keys.len()); + o["expression"] = json!(p.expr.as_ref().map(|e| e.text.clone()).unwrap_or_default()); + o["expressionEnabled"] = json!(p.has_expression()); + o["canVaryOverTime"] = json!(!p.static_only); + o["canSetExpression"] = json!(!p.static_only && !matches!(p.ui, ParamUi::Hidden)); + o["hidden"] = json!(matches!(p.ui, ParamUi::Hidden)); + o["enabled"] = json!(true); + o["canSetEnabled"] = json!(false); + o["isModified"] = json!(p.is_animated() || p.has_expression()); + if let ParamUi::Slider { min, max, .. } = p.ui { + o["min"] = json!(min); + o["max"] = json!(max); + } + if let ParamUi::Popup { options } = &p.ui { + o["options"] = json!(options); + } + o["ui"] = json!(match &p.ui { + ParamUi::Percent => "%", + ParamUi::Angle => "degrees", + ParamUi::Pixels | ParamUi::Point | ParamUi::Point3 => "pixels", + _ => "", + }); + // Position with Separate Dimensions. + o["separated"] = json!(node.match_id() == "position" && parent_group.match_id == "transform" && parent_group.get("positionX").is_some()); + o["separationFollower"] = json!(matches!(node.match_id(), "positionX" | "positionY" | "positionZ")); + } + } + Ok(o) +} + +fn node(s: &Session, a: &J) -> R { + let (_, c, l) = layer_of(s, a)?; + node_json(c, l, u(a, "uid").unwrap_or(0)) +} + +/// Normalized name for loose lookups: `Anchor Point`, `anchorPoint`, `anchor_point` → `anchorpoint`. +fn loose(s: &str) -> String { + s.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase() +} + +/// AE-style aliases for our match ids (attribute names scripts use). +fn alias(k: &str) -> Option<&'static str> { + Some(match k { + "anchorpoint" => "anchor", + "pointofinterest" => "poi", + "xrotation" => "rotationX", + "yrotation" => "rotationY", + "zrotation" => "rotation", + "xposition" => "positionX", + "yposition" => "positionY", + "zposition" => "positionZ", + "effect" | "effects" => "effects", + "mask" | "masks" => "masks", + "content" | "contents" => "contents", + "maskshape" | "maskpath" => "path", + "maskfeather" => "feather", + "maskopacity" => "opacity", + "maskexpansion" | "maskoffset" => "expansion", + "sourcetext" => "sourceText", + "timeremap" => "timeRemap", + "audiolevels" => "levels", + _ => return None, + }) +} + +fn child(s: &Session, a: &J) -> R { + let (_, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").unwrap_or(0); + let g: &PropGroup = if uid == 0 || uid == l.props.uid { &l.props } else { l.props.find_group(uid).ok_or("not a property group")? }; + let key = a.get("key").cloned().unwrap_or(J::Null); + let found = match &key { + J::Number(n) => n.as_u64().and_then(|i| g.children.get((i as usize).checked_sub(1)?)), + J::String(k) => { + let chain_of = |n: &Node| -> String { + let mut ch = if uid == 0 || uid == l.props.uid { vec![] } else { chain(l, uid).unwrap_or_default() }; + ch.push(n); + matchnames::node(&ch) + }; + let lk = loose(k); + g.children + .iter() + .find(|n| chain_of(n) == *k) + .or_else(|| g.children.iter().find(|n| n.name() == k)) + .or_else(|| g.children.iter().find(|n| n.match_id() == k)) + .or_else(|| g.children.iter().find(|n| n.name().eq_ignore_ascii_case(k))) + .or_else(|| g.children.iter().find(|n| loose(n.name()) == lk || loose(n.match_id()) == lk)) + .or_else(|| alias(&lk).and_then(|m| g.children.iter().find(|n| n.match_id() == m))) + } + _ => None, + }; + match found { + Some(n) => node_json(c, l, n.uid()), + None => Ok(J::Null), + } +} + +/// A value as the object model shows it: popups 1-based, checkboxes 0/1, layer references as +/// layer indices, paths as `{vertices, inTangents, outTangents, closed}`, text as the document. +fn js_value(c: &Comp, v: &KV) -> J { + match v { + KV::Bool(b) => json!(if *b { 1 } else { 0 }), + KV::Enum(i) => json!(*i + 1), + KV::Layer(l) => json!(l.and_then(|id| c.index_of(LayerId(id))).unwrap_or(0)), + KV::Path(p) => json!({"vertices": p.vertices, "inTangents": p.in_tangents, "outTangents": p.out_tangents, "closed": p.closed}), + KV::Text(d) => text_json(d), + v => v.to_json(), + } +} + +/// A text document for `TextDocument`: the serialized document plus the first character's style. +fn text_json(d: &TextDoc) -> J { + let mut doc = serde_json::to_value(d).unwrap_or(J::Null); + let st = d.style_at(0); + if let Some(o) = doc.as_object_mut() { + o.insert("first".into(), effectcraft_keyframe::text_doc::char_style_json(&st)); + o.insert("para".into(), effectcraft_keyframe::text_doc::para_style_json(&d.para(0))); + } + doc +} + +fn value(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists (it was deleted)")?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let pre = a.get("pre").and_then(J::as_bool).unwrap_or(false); + let v = if pre || !p.has_expression() { + p.value_at(l.layer_time(t)) + } else { + let mut ctx = EvalCtx::new(&s.project, cid, c, t); + ctx.expr = s.expr.as_deref(); + ctx.value(l, p) + }; + Ok(js_value(c, &v)) +} + +fn interp(i: Interp) -> &'static str { + match i { + Interp::Linear => "LINEAR", + Interp::Bezier => "BEZIER", + Interp::Hold => "HOLD", + } +} + +fn keys(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists (it was deleted)")?; + let sel: Vec = s.state.selected_keys.iter().filter(|k| k.layer == l.id && k.prop == uid).map(|k| k.time).collect(); + let _ = cid; + let ease = |e: &[effectcraft_keyframe::Ease], n: usize| -> J { + let n = n.max(1); + // Influence is stored as a fraction; scripts see percent. + let mut v: Vec = e.iter().map(|x| json!({"speed": x.speed, "influence": x.influence * 100.0})).collect(); + let d = e.first().cloned().unwrap_or_default(); + while v.len() < n { + v.push(json!({"speed": d.speed, "influence": d.influence * 100.0})); + } + J::Array(v) + }; + let dims = if p.spatial { 1 } else { p.value.dims().max(1) }; + let out: Vec = p + .keys + .iter() + .map(|k| { + json!({ + "time": l.comp_time(k.time).seconds(), + "layerTime": k.time.seconds(), + "value": js_value(c, &k.value), + "inInterp": interp(k.in_interp), "outInterp": interp(k.out_interp), + "inEase": ease(&k.in_ease, dims), "outEase": ease(&k.out_ease, dims), + "continuous": k.continuous, "autoBezier": k.auto_bezier, "roving": k.roving, + "spatialIn": k.spatial_in, "spatialOut": k.spatial_out, + "spatialContinuous": k.spatial_continuous, "spatialAutoBezier": k.spatial_auto, + "selected": sel.contains(&k.time), + }) + }) + .collect(); + Ok(J::Array(out)) +} + +/// Apply `layer.setText` attribute keys to the Source Text document at `time` and return the new +/// document (for `prop.set` / `prop.addKey`). +fn textdoc(s: &Session, a: &J) -> R { + let (cid, _, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists")?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let KV::Text(doc) = p.value_at(l.layer_time(t)) else { return Err("not a Source Text property".into()) }; + let mut d = *doc; + if let Some(attrs) = a.get("attrs").and_then(J::as_object) { + if let Some(text) = attrs.get("text").and_then(J::as_str) { + d.set_text(text); + } + for (k, v) in attrs { + if k == "text" || v.is_null() { + continue; + } + match d.set_attr(k, v, None) { + Ok(true) => {} + Ok(false) => return Err(format!("TextDocument: unknown attribute `{k}`")), + Err(e) => return Err(format!("TextDocument.{k}: {e}")), + } + } + } + serde_json::to_value(&d).map_err(|e| e.to_string()) +} + +fn markers(s: &Session, a: &J) -> R { + let (_, c) = comp_of(s, a)?; + let (list, to_comp): (&[effectcraft_project::Marker], Option<&Layer>) = match u(a, "layer") { + Some(lid) => { + let l = c.layer(LayerId(lid)).ok_or("the layer no longer exists")?; + (&l.markers, Some(l)) + } + None => (&c.markers, None), + }; + let out: Vec = list + .iter() + .map(|m| { + let t = to_comp.map(|l| l.comp_time(m.time)).unwrap_or(m.time); + json!({ + "time": t.seconds(), "duration": m.duration.seconds(), "comment": m.comment, "chapter": m.chapter, "url": m.url, + "frameTarget": m.frame_target, "label": label_index(m.label), "protectedRegion": m.protected, + "cuePointName": m.cue_point.as_ref().map(|c| c.name.clone()).unwrap_or_default(), + "eventCuePoint": m.cue_point.as_ref().is_some_and(|c| !c.navigation), + "params": m.cue_point.as_ref().map(|c| c.params.clone()).unwrap_or_default(), + }) + }) + .collect(); + Ok(J::Array(out)) +} + +fn source_rect(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let mut ctx = EvalCtx::new(&s.project, cid, c, t); + ctx.expr = s.expr.as_deref(); + let r = match &l.source { + LayerSource::Text => { + let paths: Vec<_> = effectcraft_render::text::glyph_paths(&ctx, l).into_iter().map(|(p, _)| p).collect(); + effectcraft_path::bounds(&paths).map(|b| [b.y0, b.x0, b.width(), b.height()]).unwrap_or([0.0; 4]) + } + LayerSource::Shape => match l.props.sub("contents") { + Some(contents) => { + let buf = effectcraft_render::shapes::render(&ctx, l, contents, 1.0); + if buf.img.width <= 4 { [0.0; 4] } else { [2.0 - buf.offset[1], 2.0 - buf.offset[0], buf.img.width as f64 - 4.0, buf.img.height as f64 - 4.0] } + } + None => [0.0; 4], + }, + _ => { + let (w, h) = effectcraft_render::source_size(&s.project, l); + [0.0, 0.0, w as f64, h as f64] + } + }; + Ok(json!({"top": r[0], "left": r[1], "width": r[2], "height": r[3]})) +} + +/// What `PropertyGroup.addProperty(name)` adds to the group `uid`. +fn resolve_add(s: &Session, a: &J) -> R { + let (_, _, l) = layer_of(s, a)?; + let uid = u(a, "uid").unwrap_or(0); + let name = a.get("name").and_then(J::as_str).unwrap_or(""); + let g: &PropGroup = if uid == 0 || uid == l.props.uid { &l.props } else { l.props.find_group(uid).ok_or("not a property group")? }; + let in_contents = g.match_id == "contents"; + let in_group = l.props.sub("contents").is_some_and(|c| c.uid == g.uid || c.find_group(g.uid).is_some()); + Ok(match g.match_id.as_str() { + "effects" => { + let id = matchnames::effect_from_match(name).map(str::to_string).or_else(|| effectcraft_engine::effects::lookup(name).map(|e| e.id.to_string())); + match id { + Some(id) => json!({"kind": "effect", "effect": id}), + None => json!({"error": format!("Can not add a property with name \"{name}\" to this PropertyGroup (unknown effect)")}), + } + } + "masks" => { + if name == "ADBE Mask Atom" || loose(name) == "mask" { + json!({"kind": "mask"}) + } else { + json!({"error": format!("Can not add \"{name}\" to Masks (use \"ADBE Mask Atom\")")}) + } + } + "animators" => json!({"kind": "animator"}), + _ if in_contents && in_group => match matchnames::shape_kind(name) { + Some(k) => { + json!({"kind": "shape", "shape": k, "group": if l.props.sub("contents").is_some_and(|c| c.uid == g.uid) { J::Null } else { json!(g.uid) }}) + } + None => json!({"error": format!("Can not add \"{name}\" to Contents")}), + }, + "properties" if l.props.group("text/animators").is_some_and(|a| a.find_group(g.uid).is_some()) => { + let animator = l.props.group("text/animators").and_then(|a| a.groups().find(|x| x.find_group(g.uid).is_some()).map(|x| x.uid)); + match text_animator_property(name) { + Some(k) => json!({"kind": "animatorProperty", "property": k, "animator": animator}), + None => json!({"error": format!("Can not add \"{name}\" to an animator's Properties")}), + } + } + "selectors" if l.props.group("text/animators").is_some_and(|a| a.find_group(g.uid).is_some()) => { + let animator = l.props.group("text/animators").and_then(|a| a.groups().find(|x| x.find_group(g.uid).is_some()).map(|x| x.uid)); + let kind = match name { + "ADBE Text Wiggly Selector" => "wiggly", + "ADBE Text Expressible Selector" => "expression", + n if loose(n).contains("wiggly") => "wiggly", + n if loose(n).contains("expression") => "expression", + _ => "range", + }; + json!({"kind": "selector", "selector": kind, "animator": animator}) + } + _ => json!({"error": format!("Can not add a property to \"{}\"", g.name)}), + }) +} + +/// A text animator property (`layer.addTextAnimatorProperty` key) from an After Effects match +/// name or a display name. +fn text_animator_property(name: &str) -> Option<&'static str> { + const AE: &[(&str, &str)] = &[ + ("ADBE Text Anchor Point 3D", "anchor"), + ("ADBE Text Position 3D", "position"), + ("ADBE Text Scale 3D", "scale"), + ("ADBE Text Skew", "skew"), + ("ADBE Text Rotation", "rotation"), + ("ADBE Text Opacity", "opacity"), + ("ADBE Text Fill Color", "fillColor"), + ("ADBE Text Fill Hue", "fillHue"), + ("ADBE Text Fill Saturation", "fillSaturation"), + ("ADBE Text Fill Brightness", "fillBrightness"), + ("ADBE Text Fill Opacity", "fillOpacity"), + ("ADBE Text Stroke Color", "strokeColor"), + ("ADBE Text Stroke Hue", "strokeHue"), + ("ADBE Text Stroke Saturation", "strokeSaturation"), + ("ADBE Text Stroke Brightness", "strokeBrightness"), + ("ADBE Text Stroke Opacity", "strokeOpacity"), + ("ADBE Text Stroke Width", "strokeWidth"), + ("ADBE Text Tracking Amount", "tracking"), + ("ADBE Text Line Anchor", "lineAnchor"), + ("ADBE Text Line Spacing", "lineSpacing"), + ("ADBE Text Character Offset", "characterOffset"), + ("ADBE Text Character Replace", "characterValue"), + ("ADBE Text Blur", "blur"), + ]; + if let Some((_, k)) = AE.iter().find(|(m, _)| m.eq_ignore_ascii_case(name)) { + return Some(k); + } + let n = loose(name); + AE.iter().map(|(_, k)| *k).find(|k| loose(k) == n || (n == "anchorpoint" && *k == "anchor") || (n == "trackingamount" && *k == "tracking")) +} + +/// `app.findMenuCommandId(name)`: 1-based index of the command whose label matches (menu items +/// first), 0 when none does. +fn menu_command_id(name: &str) -> J { + let norm = |s: &str| s.trim().trim_end_matches("...").trim_end_matches('…').trim().to_ascii_lowercase(); + let n = norm(name); + let specs = effectcraft_engine::command_specs(); + let pick = specs + .iter() + .position(|c| !c.menu.is_empty() && norm(c.label) == n) + .or_else(|| specs.iter().position(|c| norm(c.label) == n)) + .or_else(|| specs.iter().position(|c| c.id == name)); + json!(pick.map(|i| i + 1).unwrap_or(0)) +} diff --git a/crates/script/src/prelude.js b/crates/script/src/prelude.js new file mode 100644 index 000000000..4f3599264 --- /dev/null +++ b/crates/script/src/prelude.js @@ -0,0 +1,1689 @@ +// EffectCraft scripting: an After Effects-style object model over the engine. +// +// Written from the behaviour the After Effects Scripting Guide documents (object names, +// attributes, methods). Reads go through `__query(kind, json)`; every edit runs an engine command +// through `__exec(id, json)`, so it is undoable and identical to the UI's action. + +var __ecGlobal = this; + +function __call(id, p) { + var r = __exec(id, JSON.stringify(p || {})); + return r === undefined || r === "" ? null : JSON.parse(r); +} +function __get(kind, a) { + return JSON.parse(__query(kind, JSON.stringify(a || {}))); +} +function __err(msg) { + return new Error(msg); +} +function __num(v, what) { + var n = Number(v); + if (typeof v === "boolean" || isNaN(n)) throw __err((what || "value") + " must be a number"); + return n; +} +function __str(v) { + if (v === undefined || v === null) return ""; + if (v instanceof File || v instanceof Folder) return v.fsName; + return String(v); +} + +// ------------------------------------------------------------------ enums + +function __enum(name, keys, base) { + var e = {}; + for (var i = 0; i < keys.length; i++) e[keys[i]] = base + i; + e.__name = name; + e.__keys = keys; + e.__base = base; + return e; +} +function __enumKey(e, v) { + var i = v - e.__base; + return i >= 0 && i < e.__keys.length ? e.__keys[i] : null; +} + +var BlendingMode = __enum("BlendingMode", [ + "NORMAL", "DISSOLVE", "DANCING_DISSOLVE", "DARKEN", "MULTIPLY", "COLOR_BURN", "CLASSIC_COLOR_BURN", "LINEAR_BURN", + "DARKER_COLOR", "ADD", "LIGHTEN", "SCREEN", "COLOR_DODGE", "CLASSIC_COLOR_DODGE", "LINEAR_DODGE", "LIGHTER_COLOR", + "OVERLAY", "SOFT_LIGHT", "HARD_LIGHT", "LINEAR_LIGHT", "VIVID_LIGHT", "PIN_LIGHT", "HARD_MIX", "DIFFERENCE", + "CLASSIC_DIFFERENCE", "EXCLUSION", "SUBTRACT", "DIVIDE", "HUE", "SATURATION", "COLOR", "LUMINOSITY", "STENCIL_ALPHA", + "STENCIL_LUMA", "SILHOUETE_ALPHA", "SILHOUETTE_LUMA", "ALPHA_ADD", "LUMINESCENT_PREMUL"], 5212); +BlendingMode.SILHOUETTE_ALPHA = BlendingMode.SILHOUETE_ALPHA; +var KeyframeInterpolationType = __enum("KeyframeInterpolationType", ["LINEAR", "BEZIER", "HOLD"], 6612); +var TrackMatteType = __enum("TrackMatteType", ["NO_TRACK_MATTE", "ALPHA", "ALPHA_INVERTED", "LUMA", "LUMA_INVERTED"], 5012); +var LightType = __enum("LightType", ["PARALLEL", "SPOT", "POINT", "AMBIENT", "ENVIRONMENT"], 4412); +var ParagraphJustification = __enum("ParagraphJustification", [ + "LEFT_JUSTIFY", "CENTER_JUSTIFY", "RIGHT_JUSTIFY", "FULL_JUSTIFY_LASTLINE_LEFT", "FULL_JUSTIFY_LASTLINE_CENTER", + "FULL_JUSTIFY_LASTLINE_RIGHT", "FULL_JUSTIFY_LASTLINE_FULL", "MULTIPLE_JUSTIFICATIONS"], 7413); +var PropertyValueType = __enum("PropertyValueType", [ + "NO_VALUE", "ThreeD_SPATIAL", "ThreeD", "TwoD_SPATIAL", "TwoD", "OneD", "COLOR", "CUSTOM_VALUE", "MARKER", + "LAYER_INDEX", "MASK_INDEX", "SHAPE", "TEXT_DOCUMENT"], 6412); +var PropertyType = __enum("PropertyType", ["PROPERTY", "INDEXED_GROUP", "NAMED_GROUP"], 6212); +var LayerQuality = __enum("LayerQuality", ["BEST", "DRAFT", "WIREFRAME"], 4612); +var LayerSamplingQuality = __enum("LayerSamplingQuality", ["BILINEAR", "BICUBIC"], 4812); +var FrameBlendingType = __enum("FrameBlendingType", ["NO_FRAME_BLEND", "FRAME_MIX", "PIXEL_MOTION"], 4012); +var AutoOrientType = __enum("AutoOrientType", ["NO_AUTO_ORIENT", "ALONG_PATH", "CAMERA_OR_POINT_OF_INTEREST", "CHARACTERS_TOWARD_CAMERA"], 4212); +var MaskMode = __enum("MaskMode", ["NONE", "ADD", "SUBTRACT", "INTERSECT", "LIGHTEN", "DARKEN", "DIFFERENCE"], 6812); +var RQItemStatus = __enum("RQItemStatus", ["WILL_CONTINUE", "NEEDS_OUTPUT", "UNQUEUED", "QUEUED", "RENDERING", "USER_STOPPED", "ERR_STOPPED", "DONE"], 3012); +var CloseOptions = __enum("CloseOptions", ["DO_NOT_SAVE_CHANGES", "PROMPT_TO_SAVE_CHANGES", "SAVE_CHANGES"], 1212); +var ImportAsType = __enum("ImportAsType", ["COMP_CROPPED_LAYERS", "FOOTAGE", "COMP", "PROJECT"], 3812); +var TimeDisplayType = __enum("TimeDisplayType", ["FRAMES", "TIMECODE"], 2012); +var LogType = __enum("LogType", ["ERRORS_ONLY", "ERRORS_AND_SETTINGS", "ERRORS_AND_PER_FRAME_INFO"], 3212); +var KeyframeLabel = __enum("KeyframeLabel", ["NONE"], 0); + +var __blendNames = {}; +(function () { + var labels = ["Normal", "Dissolve", "Dancing Dissolve", "Darken", "Multiply", "Color Burn", "Classic Color Burn", + "Linear Burn", "Darker Color", "Add", "Lighten", "Screen", "Color Dodge", "Classic Color Dodge", "Linear Dodge", + "Lighter Color", "Overlay", "Soft Light", "Hard Light", "Linear Light", "Vivid Light", "Pin Light", "Hard Mix", + "Difference", "Classic Difference", "Exclusion", "Subtract", "Divide", "Hue", "Saturation", "Color", "Luminosity", + "Stencil Alpha", "Stencil Luma", "Silhouette Alpha", "Silhouette Luma", "Alpha Add", "Luminescent Premul"]; + for (var i = 0; i < labels.length; i++) __blendNames[labels[i]] = BlendingMode.__base + i; +})(); +function __blendLabel(v) { + for (var k in __blendNames) if (__blendNames[k] === v) return k; + throw __err("Bad blendingMode value " + v); +} + +var __interpKey = { 6612: "linear", 6613: "bezier", 6614: "hold" }; +var __matteKind = { 5013: "alpha", 5014: "alphaInverted", 5015: "luma", 5016: "lumaInverted" }; +var __matteEnum = { Alpha: 5013, AlphaInverted: 5014, Luma: 5015, LumaInverted: 5016 }; +var __justKey = { 7413: "left", 7414: "center", 7415: "right", 7416: "justifyLeft", 7417: "justifyCenter", 7418: "justifyRight", 7419: "justifyAll" }; + +// ------------------------------------------------------------------ output, $ and dialogs + +function writeLn(s) { __print(__str(s)); } +function write(s) { __print(__str(s)); } +function alert(s) { __print(__str(s)); } +function confirm(s) { __print(__str(s)); return true; } +function prompt(msg, def) { __print(__str(msg)); return def === undefined ? "" : def; } +function clearOutput() {} + +var $ = { + writeln: function () { __print(Array.prototype.slice.call(arguments).map(__str).join("")); }, + write: function () { __print(Array.prototype.slice.call(arguments).map(__str).join("")); }, + sleep: function () {}, + gc: function () {}, + global: __ecGlobal, + level: 0, + engineName: "boa", + get fileName() { return __get("app").scriptName; }, + get os() { return "EffectCraft"; }, + get version() { return "EffectCraft scripting"; }, + get locale() { return "en_US"; }, + evalFile: function () { throw __err("$.evalFile is not supported: scripts can't run scripts"); }, +}; + +// ------------------------------------------------------------------ File / Folder + +function __join(a, b) { return a.replace(/[\/\\]+$/, "") + "/" + b; } + +function File(path) { + if (!(this instanceof File)) return new File(path); + this.__path = path === undefined ? "" : __str(path); + this.__buf = null; + this.__mode = null; + this.__pos = 0; + this.encoding = "UTF-8"; + this.lineFeed = "Unix"; + this.error = ""; +} +File.prototype = { + constructor: File, + get fsName() { return this.__path; }, + get fullName() { return this.__path; }, + get absoluteURI() { return this.__path; }, + get name() { return this.__path.split(/[\/\\]/).pop(); }, + get displayName() { return this.name; }, + get path() { var p = this.__path.split(/[\/\\]/); p.pop(); return p.join("/"); }, + get parent() { return new Folder(this.path); }, + get exists() { return JSON.parse(__file("exists", this.__path)) === true; }, + get length() { return this.__buf !== null ? this.__buf.length : (this.exists ? JSON.parse(__file("read", this.__path)).length : 0); }, + get eof() { return this.__buf === null || this.__pos >= this.__buf.length; }, + toString: function () { return this.__path; }, + open: function (mode) { + mode = (mode || "r").charAt(0).toLowerCase(); + this.__mode = mode; + this.__pos = 0; + if (mode === "r" || mode === "e") { + // Throws when the location is off limits (the scripting file gate). + if (!JSON.parse(__file("access", this.__path))) { + if (mode === "r") { this.error = "File not found"; return false; } + this.__buf = ""; + return true; + } + this.__buf = JSON.parse(__file("read", this.__path)); + } else if (mode === "a") { + this.__buf = this.exists ? JSON.parse(__file("read", this.__path)) : ""; + this.__pos = this.__buf.length; + } else { + // Check the gate now, like After Effects refusing to open a file for writing. + JSON.parse(__file("write", this.__path, "")); + this.__buf = ""; + } + return true; + }, + read: function (n) { + if (this.__buf === null) this.open("r"); + if (this.__buf === null) return ""; + var s = n === undefined ? this.__buf.substring(this.__pos) : this.__buf.substr(this.__pos, n); + this.__pos += s.length; + return s; + }, + readln: function () { + if (this.__buf === null) this.open("r"); + if (this.__buf === null) return ""; + var i = this.__buf.indexOf("\n", this.__pos); + var end = i < 0 ? this.__buf.length : i; + var s = this.__buf.substring(this.__pos, end).replace(/\r$/, ""); + this.__pos = i < 0 ? this.__buf.length : i + 1; + return s; + }, + write: function () { + if (this.__mode !== "w" && this.__mode !== "a" && this.__mode !== "e") throw __err("File is not open for writing"); + var s = Array.prototype.slice.call(arguments).map(__str).join(""); + this.__buf = this.__buf.substring(0, this.__pos) + s + this.__buf.substring(this.__pos + s.length); + this.__pos += s.length; + return true; + }, + writeln: function () { return this.write(Array.prototype.slice.call(arguments).map(__str).join("") + "\n"); }, + seek: function (p) { this.__pos = p; return true; }, + tell: function () { return this.__pos; }, + close: function () { + if ((this.__mode === "w" || this.__mode === "a" || this.__mode === "e") && this.__buf !== null) { + __file("write", this.__path, this.__buf); + } + this.__buf = null; + this.__mode = null; + return true; + }, + remove: function () { return JSON.parse(__file("remove", this.__path)); }, + copy: function (to) { + var data = JSON.parse(__file("read", this.__path)); + __file("write", __str(to), data); + return true; + }, + rename: function () { throw __err("File.rename is not supported"); }, + execute: function () { throw __err("scripts can't execute files"); }, + openDlg: function () { return null; }, + saveDlg: function () { return null; }, +}; +File.openDialog = function () { return null; }; +File.saveDialog = function () { return null; }; +File.decode = function (s) { return decodeURIComponent(s); }; +File.encode = function (s) { return encodeURIComponent(s); }; +File.fs = "EffectCraft"; + +function Folder(path) { + if (!(this instanceof Folder)) return new Folder(path); + this.__path = path === undefined ? "" : __str(path); +} +Folder.prototype = { + constructor: Folder, + get fsName() { return this.__path; }, + get fullName() { return this.__path; }, + get absoluteURI() { return this.__path; }, + get name() { return this.__path.replace(/[\/\\]+$/, "").split(/[\/\\]/).pop(); }, + get displayName() { return this.name; }, + get path() { var p = this.__path.replace(/[\/\\]+$/, "").split(/[\/\\]/); p.pop(); return p.join("/"); }, + get parent() { return new Folder(this.path); }, + get exists() { return JSON.parse(__file("isDir", this.__path)) === true; }, + toString: function () { return this.__path; }, + create: function () { return JSON.parse(__file("mkdir", this.__path)); }, + remove: function () { return JSON.parse(__file("remove", this.__path)); }, + getFiles: function (mask) { + var names = JSON.parse(__file("list", this.__path)); + var out = []; + for (var i = 0; i < names.length; i++) { + var n = names[i]; + if (typeof mask === "string" && mask !== "*") { + var re = new RegExp("^" + mask.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".") + "$", "i"); + if (!re.test(n.split(/[\/\\]/).pop())) continue; + } + var isDir = JSON.parse(__file("isDir", n)); + var f = isDir ? new Folder(n) : new File(n); + if (typeof mask === "function" && !mask(f)) continue; + out.push(f); + } + return out; + }, + selectDlg: function () { return null; }, +}; +Folder.selectDialog = function () { return null; }; +Object.defineProperty(Folder, "current", { get: function () { var d = __get("app").projectDir; return d ? new Folder(d) : null; } }); +Object.defineProperty(Folder, "temp", { get: function () { return null; } }); +Object.defineProperty(Folder, "desktop", { get: function () { return null; } }); +Object.defineProperty(Folder, "myDocuments", { get: function () { return null; } }); + +function Socket() { __file("network", ""); } +var system = { callSystem: function () { throw __err("scripts can't run system commands"); } }; + +// ------------------------------------------------------------------ helpers + +// Unknown attributes of property groups and layers resolve to child properties, like After +// Effects (`layer.transform.position`, `layer.Effects`, `effect.blurriness`). +var __attr = { + get: function (t, k, r) { + if (typeof k !== "string" || k in t || k.charAt(0) === "_" || k === "toJSON" || k === "then") return Reflect.get(t, k, r); + var c = t.property(k); + return c === null ? undefined : c; + }, +}; + +// 1-based collections with `[n]` access. +var __coll = { + get: function (t, k, r) { + if (typeof k === "string" && /^[0-9]+$/.test(k)) return t.__at(parseInt(k, 10)); + return Reflect.get(t, k, r); + }, +}; + +function KeyframeEase(speed, influence) { + if (!(this instanceof KeyframeEase)) return new KeyframeEase(speed, influence); + this.speed = speed === undefined ? 0 : speed; + this.influence = influence === undefined ? 16.666666667 : influence; +} + +function Shape() { + if (!(this instanceof Shape)) return new Shape(); + this.vertices = []; + this.inTangents = []; + this.outTangents = []; + this.closed = true; + this.featherSegLocs = []; + this.featherRelSegLocs = []; + this.featherRadii = []; + this.featherInterps = []; + this.featherTensions = []; + this.featherTypes = []; + this.featherRelCornerAngles = []; +} + +function MarkerValue(comment, chapter, url, frameTarget, cuePointName, params) { + if (!(this instanceof MarkerValue)) return new MarkerValue(comment, chapter, url, frameTarget, cuePointName, params); + this.comment = comment === undefined ? "" : String(comment); + this.chapter = chapter === undefined ? "" : String(chapter); + this.url = url === undefined ? "" : String(url); + this.frameTarget = frameTarget === undefined ? "" : String(frameTarget); + this.cuePointName = cuePointName === undefined ? "" : String(cuePointName); + this.duration = 0; + this.eventCuePoint = false; + this.label = 0; + this.protectedRegion = false; + this.__params = []; + if (params) this.setParameters(params); +} +MarkerValue.prototype.getParameters = function () { + var o = {}; + for (var i = 0; i < this.__params.length; i++) o[this.__params[i][0]] = this.__params[i][1]; + return o; +}; +MarkerValue.prototype.setParameters = function (o) { + this.__params = []; + for (var k in o) this.__params.push([k, String(o[k])]); +}; + +// ------------------------------------------------------------------ TextDocument + +// `layer.setText` keys of each TextDocument attribute. +var __textKeys = { + font: "font", fontStyle: "style", fontSize: "size", fillColor: "fill", strokeColor: "stroke", strokeWidth: "strokeWidth", + applyFill: "applyFill", applyStroke: "applyStroke", strokeOverFill: "strokeOverFill", tracking: "tracking", + baselineShift: "baselineShift", horizontalScale: "hScale", verticalScale: "vScale", fauxBold: "fauxBold", + fauxItalic: "fauxItalic", allCaps: "allCaps", smallCaps: "smallCaps", tsume: "tsume", superscript: "superscript", + subscript: "subscript", ligature: "ligatures", startIndent: "indentLeft", endIndent: "indentRight", + firstLineIndent: "indentFirst", spaceBefore: "spaceBefore", spaceAfter: "spaceAfter", +}; + +function TextDocument(text) { + if (!(this instanceof TextDocument)) return new TextDocument(text); + this.__base = null; + this.__changed = {}; + this.__text = text === undefined ? "" : String(text); + this.__textSet = text !== undefined; +} +TextDocument.__from = function (doc) { + var t = new TextDocument(); + t.__base = doc; + t.__text = doc.text; + t.__textSet = false; + return t; +}; +TextDocument.prototype = { + constructor: TextDocument, + get text() { return this.__text; }, + set text(v) { this.__text = String(v); this.__textSet = true; }, + __style: function (k, d) { + if (k in this.__changed) return this.__changed[k]; + if (this.__base && this.__base.first && k in this.__base.first) return this.__base.first[k]; + if (this.__base && this.__base.para && k in this.__base.para) return this.__base.para[k]; + return d; + }, + __set: function (k, v) { this.__changed[k] = v; }, + get font() { return this.__style("font", "Inter"); }, + set font(v) { this.__set("font", String(v)); }, + get fontFamily() { return this.font; }, + get fontStyle() { return this.__style("style", "Regular"); }, + set fontStyle(v) { this.__set("style", String(v)); }, + get fontSize() { return this.__style("size", 72); }, + set fontSize(v) { this.__set("size", __num(v, "fontSize")); }, + get fillColor() { var c = this.__style("fill", [1, 1, 1, 1]); return [c[0], c[1], c[2]]; }, + set fillColor(v) { this.__set("fill", [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]); }, + get strokeColor() { var c = this.__style("stroke", [0, 0, 0, 1]); return [c[0], c[1], c[2]]; }, + set strokeColor(v) { this.__set("stroke", [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]); }, + get strokeWidth() { return this.__style("strokeWidth", 0); }, + set strokeWidth(v) { this.__set("strokeWidth", __num(v, "strokeWidth")); }, + get applyFill() { return this.__style("applyFill", true); }, + set applyFill(v) { this.__set("applyFill", !!v); }, + get applyStroke() { return this.__style("applyStroke", false); }, + set applyStroke(v) { this.__set("applyStroke", !!v); }, + get strokeOverFill() { return "strokeOverFill" in this.__changed ? this.__changed.strokeOverFill : (this.__base ? this.__base.stroke_over_fill : true); }, + set strokeOverFill(v) { this.__set("strokeOverFill", !!v); }, + get tracking() { return this.__style("tracking", 0); }, + set tracking(v) { this.__set("tracking", __num(v, "tracking")); }, + get leading() { var l = this.__style("leading", "auto"); return l === "auto" ? this.fontSize * 1.2 : l; }, + set leading(v) { this.__set("leading", __num(v, "leading")); }, + get autoLeading() { return this.__style("leading", "auto") === "auto"; }, + set autoLeading(v) { if (v) this.__set("leading", "auto"); }, + get justification() { + var j = this.__style("justify", "left"); + for (var k in __justKey) if (__justKey[k] === j) return Number(k); + return ParagraphJustification.LEFT_JUSTIFY; + }, + set justification(v) { + var k = __justKey[v]; + if (!k) throw __err("Bad justification value " + v); + this.__set("justify", k); + }, + get baselineShift() { return this.__style("baselineShift", 0); }, + set baselineShift(v) { this.__set("baselineShift", __num(v)); }, + get horizontalScale() { return this.__style("hScale", 100) / 100; }, + set horizontalScale(v) { this.__set("hScale", __num(v) * 100); }, + get verticalScale() { return this.__style("vScale", 100) / 100; }, + set verticalScale(v) { this.__set("vScale", __num(v) * 100); }, + get fauxBold() { return this.__style("fauxBold", false); }, + set fauxBold(v) { this.__set("fauxBold", !!v); }, + get fauxItalic() { return this.__style("fauxItalic", false); }, + set fauxItalic(v) { this.__set("fauxItalic", !!v); }, + get allCaps() { return this.__style("allCaps", false); }, + set allCaps(v) { this.__set("allCaps", !!v); }, + get smallCaps() { return this.__style("smallCaps", false); }, + set smallCaps(v) { this.__set("smallCaps", !!v); }, + get superscript() { return this.__style("baseline", "normal") === "superscript"; }, + set superscript(v) { this.__set("superscript", !!v); }, + get subscript() { return this.__style("baseline", "normal") === "subscript"; }, + set subscript(v) { this.__set("subscript", !!v); }, + get tsume() { return this.__style("tsume", 0); }, + set tsume(v) { this.__set("tsume", __num(v)); }, + get boxText() { return !!(this.__base && this.__base.box_size); }, + get pointText() { return !this.boxText; }, + get boxTextSize() { return this.__base && this.__base.box_size ? this.__base.box_size.slice() : undefined; }, + set boxTextSize(v) { + var p = this.boxTextPos || [0, 0]; + this.__set("box", [p[0], p[1], v[0], v[1]]); + }, + get boxTextPos() { return this.__base && this.__base.box_size ? this.__base.box_pos.slice() : undefined; }, + get startIndent() { return this.__style("indentLeft", 0); }, + set startIndent(v) { this.__set("indentLeft", __num(v)); }, + get endIndent() { return this.__style("indentRight", 0); }, + set endIndent(v) { this.__set("indentRight", __num(v)); }, + get firstLineIndent() { return this.__style("indentFirst", 0); }, + set firstLineIndent(v) { this.__set("indentFirst", __num(v)); }, + get spaceBefore() { return this.__style("spaceBefore", 0); }, + set spaceBefore(v) { this.__set("spaceBefore", __num(v)); }, + get spaceAfter() { return this.__style("spaceAfter", 0); }, + set spaceAfter(v) { this.__set("spaceAfter", __num(v)); }, + resetCharStyle: function () { + var d = { font: "Inter", style: "Regular", size: 72, fill: [1, 1, 1, 1], stroke: [0, 0, 0, 1], strokeWidth: 0, applyFill: true, + applyStroke: false, tracking: 0, leading: "auto", baselineShift: 0, hScale: 100, vScale: 100, tsume: 0, fauxBold: false, + fauxItalic: false, allCaps: false, smallCaps: false }; + for (var k in d) this.__changed[k] = d[k]; + }, + resetParagraphStyle: function () { + var d = { justify: "left", indentLeft: 0, indentRight: 0, indentFirst: 0, spaceBefore: 0, spaceAfter: 0 }; + for (var k in d) this.__changed[k] = d[k]; + }, + toString: function () { return this.__text; }, + // `layer.setText` attributes to apply. + __attrs: function () { + var a = {}; + for (var k in this.__changed) a[k] = this.__changed[k]; + if (this.__textSet || !this.__base || this.__text !== this.__base.text) a.text = this.__text; + return a; + }, +}; + +// ------------------------------------------------------------------ properties + +function __propOf(layer, info) { + if (!info) return null; + if (info.isLayer) return layer; + if (info.kind === "group") return info.isMask ? new MaskPropertyGroup(layer, info.uid) : new PropertyGroup(layer, info.uid); + return new Property(layer, info.uid); +} + +function PropertyBase() {} +PropertyBase.prototype = { + __info: function () { return __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid }); }, + __ref: function (p) { + p = p || {}; + p.comp = this.__layer.__comp; + p.layer = this.__layer.__id; + p.prop = this.__uid; + return p; + }, + get name() { return this.__info().name; }, + set name(v) { + var i = this.__info(); + if (!i.instance) throw __err("Can not set the name of \"" + i.name + "\": it is not an indexed group member"); + __call("prop.renameGroup", this.__ref({ name: String(v) })); + }, + get matchName() { return this.__info().matchName; }, + get propertyIndex() { return this.__info().index; }, + get propertyDepth() { return this.__info().depth; }, + get propertyType() { return PropertyType[this.__info().propertyType]; }, + get parentProperty() { + var i = this.__info(); + if (i.depth <= 1) return this.__layer; + return __propOf(this.__layer, __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: i.parent })); + }, + get isModified() { return !!this.__info().isModified; }, + get canSetEnabled() { return !!this.__info().canSetEnabled; }, + get enabled() { return this.__info().enabled !== false; }, + set enabled(v) { + if (!this.canSetEnabled) throw __err("Can not set enabled on \"" + this.name + "\""); + __call("prop.setGroupEnabled", this.__ref({ value: !!v })); + }, + get active() { return this.enabled && this.__layer.enabled; }, + get elided() { return false; }, + get isEffect() { return !!this.__info().isEffect; }, + get isMask() { return !!this.__info().isMask; }, + get selected() { + var c = __get("item", { id: this.__layer.__comp }).selectedProperties; + for (var i = 0; i < c.length; i++) if (c[i][0] === this.__layer.__id && c[i][1] === this.__uid) return true; + return false; + }, + set selected(v) { + if (v) __call("prop.select", { comp: this.__layer.__comp, layer: this.__layer.__id, prop: this.__uid, add: true }); + }, + propertyGroup: function (n) { + var p = this; + n = n === undefined ? 1 : n; + for (var i = 0; i < n && p; i++) p = p.parentProperty; + return p || null; + }, + remove: function () { __call("prop.removeGroup", this.__ref()); }, + moveTo: function (i) { __call("prop.moveGroup", this.__ref({ index: __num(i, "index") })); }, + duplicate: function () { + var r = __call("prop.duplicateGroup", this.__ref()); + return __propOf(this.__layer, __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: r.prop })); + }, + toString: function () { return "[object " + this.__kind + "]"; }, +}; + +function PropertyGroup(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "PropertyGroup"; + return new Proxy(this, __attr); +} +PropertyGroup.prototype = Object.create(PropertyBase.prototype); +PropertyGroup.prototype.constructor = PropertyGroup; +Object.defineProperty(PropertyGroup.prototype, "numProperties", { get: function () { return this.__info().numProperties; } }); +PropertyGroup.prototype.property = function (key) { + if (key === undefined || key === null) throw __err("property(): missing name or index"); + var k = typeof key === "number" ? key : String(key); + return __propOf(this.__layer, __get("child", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, key: k })); +}; +PropertyGroup.prototype.canAddProperty = function (name) { + var r = __get("resolveAdd", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, name: String(name) }); + return !r.error; +}; +PropertyGroup.prototype.addProperty = function (name) { + var L = this.__layer; + var r = __get("resolveAdd", { comp: L.__comp, layer: L.__id, uid: this.__uid, name: String(name) }); + if (r.error) throw __err(r.error); + var uid; + if (r.kind === "effect") uid = __call("effect.apply", { comp: L.__comp, layers: [L.__id], effect: r.effect }).effects[0]; + else if (r.kind === "mask") uid = __call("layer.addMask", { comp: L.__comp, layer: L.__id }).mask; + else if (r.kind === "shape") { + var p = { comp: L.__comp, layer: L.__id, kind: r.shape }; + if (r.group !== null && r.group !== undefined) p.group = r.group; + uid = __call("layer.addShapeItem", p).uid; + } else if (r.kind === "animator") { + uid = __call("layer.addTextAnimator", { comp: L.__comp, layer: L.__id, properties: [] }).animator; + } else if (r.kind === "animatorProperty") { + var before = this.numProperties; + __call("layer.addTextAnimatorProperty", { comp: L.__comp, layer: L.__id, animator: r.animator, property: r.property }); + return this.property(before + 1); + } else if (r.kind === "selector") { + var n0 = this.numProperties; + __call("layer.addTextSelector", { comp: L.__comp, layer: L.__id, animator: r.animator, kind: r.selector }); + return this.property(n0 + 1); + } else throw __err("Can not add \"" + name + "\" here"); + return __propOf(L, __get("node", { comp: L.__comp, layer: L.__id, uid: uid })); +}; + +function MaskPropertyGroup(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "MaskPropertyGroup"; + return new Proxy(this, __attr); +} +MaskPropertyGroup.prototype = Object.create(PropertyGroup.prototype); +MaskPropertyGroup.prototype.constructor = MaskPropertyGroup; +(function () { + var modes = { "None": 6812, "Add": 6813, "Subtract": 6814, "Intersect": 6815, "Lighten": 6816, "Darken": 6817, "Difference": 6818 }; + function mask(o) { return { comp: o.__layer.__comp, layer: o.__layer.__id, mask: o.__uid }; } + Object.defineProperty(MaskPropertyGroup.prototype, "maskMode", { + get: function () { return modes[this.__info().maskMode]; }, + set: function (v) { + var p = mask(this); + for (var k in modes) if (modes[k] === v) p.mode = k; + if (!p.mode) throw __err("Bad MaskMode value " + v); + __call("layer.mask.mode", p); + }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "inverted", { + get: function () { return !!this.__info().inverted; }, + set: function (v) { var p = mask(this); p.value = !!v; __call("layer.mask.invert", p); }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "locked", { + get: function () { return !!this.__info().locked; }, + set: function (v) { var p = mask(this); p.value = !!v; __call("layer.mask.lock", p); }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "color", { get: function () { return this.__info().color; } }); + MaskPropertyGroup.prototype.remove = function () { __call("layer.mask.remove", mask(this)); }; +})(); + +function Property(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "Property"; + return new Proxy(this, __attr); +} +Property.prototype = Object.create(PropertyBase.prototype); +Property.prototype.constructor = Property; +Property.prototype.property = function () { return null; }; +(function () { + var P = Property.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + def("propertyValueType", function () { return PropertyValueType[this.__info().valueType]; }); + def("value", function () { return this.valueAtTime(this.__layer.containingComp.time, false); }); + def("numKeys", function () { return this.__info().numKeys; }); + def("expression", function () { return this.__info().expression; }, function (v) { + if (!this.canSetExpression) throw __err("Can not set an expression on \"" + this.name + "\""); + __call("prop.setExpression", this.__ref({ expression: String(v) })); + }); + def("expressionEnabled", function () { return !!this.__info().expressionEnabled; }, function (v) { + __call("prop.setExpression", this.__ref({ enabled: !!v })); + }); + def("expressionError", function () { + var e = this.expression; + return e ? __get("exprError", { code: e }) : ""; + }); + def("canSetExpression", function () { return !!this.__info().canSetExpression; }); + def("canVaryOverTime", function () { return !!this.__info().canVaryOverTime; }); + def("isTimeVarying", function () { var i = this.__info(); return i.numKeys > 0 || !!i.expressionEnabled; }); + def("isSpatial", function () { return !!this.__info().spatial; }); + def("unitsText", function () { return this.__info().ui; }); + def("hasMin", function () { return this.__info().min !== undefined; }); + def("hasMax", function () { return this.__info().max !== undefined; }); + def("minValue", function () { var i = this.__info(); if (i.min === undefined) throw __err("property has no minimum"); return i.min; }); + def("maxValue", function () { var i = this.__info(); if (i.max === undefined) throw __err("property has no maximum"); return i.max; }); + def("dimensionsSeparated", function () { return !!this.__info().separated; }, function (v) { + __call("prop.separateDimensions", { comp: this.__layer.__comp, layer: this.__layer.__id, value: !!v }); + }); + def("isSeparationFollower", function () { return !!this.__info().separationFollower; }); + def("isSeparationLeader", function () { return this.__info().matchId === "position" && this.__info().depth === 2; }); + def("separationDimension", function () { var m = this.__info().matchId; return m === "positionY" ? 1 : m === "positionZ" ? 2 : 0; }); + def("separationLeader", function () { return this.isSeparationFollower ? this.parentProperty.property("ADBE Position") : null; }); + def("selectedKeys", function () { + var k = this.__keys(), out = []; + for (var i = 0; i < k.length; i++) if (k[i].selected) out.push(i + 1); + return out; + }); + def("isDropdownEffect", function () { return false; }); + + P.getSeparationFollower = function (dim) { + return this.parentProperty.property(["ADBE Position_0", "ADBE Position_1", "ADBE Position_2"][dim]); + }; + P.__keys = function () { return __get("keys", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid }); }; + P.__key = function (i) { + var k = this.__keys(); + i = __num(i, "keyIndex"); + if (i < 1 || i > k.length || Math.floor(i) !== i) throw __err("keyIndex " + i + " is out of range 1.." + k.length); + return k[i - 1]; + }; + // Script value → engine JSON for `prop.set` / `prop.addKey`. + P.__toEngine = function (v, info, time) { + var t = info.valueType; + if (t === "TEXT_DOCUMENT") { + var td = v instanceof TextDocument ? v : new TextDocument(String(v)); + return __get("textdoc", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, time: time, attrs: td.__attrs() }); + } + if (t === "SHAPE") { + if (!v || !v.vertices) throw __err("value must be a Shape"); + var n = v.vertices.length; + function fit(a) { var o = []; for (var i = 0; i < n; i++) o.push(a && a[i] ? [a[i][0], a[i][1]] : [0, 0]); return o; } + return { vertices: fit(v.vertices), in_tangents: fit(v.inTangents), out_tangents: fit(v.outTangents), closed: v.closed !== false }; + } + if (t === "LAYER_INDEX") { + var idx = v && v.__id !== undefined ? v.index : __num(v); + if (!idx) return null; + return this.__layer.containingComp.layer(idx).__id; + } + if (info.type === "enum") return __num(v) - 1; + if (info.type === "bool") return typeof v === "boolean" ? v : __num(v) !== 0; + if (info.type === "color") { + if (!(v instanceof Array) || v.length < 3) throw __err("value must be an array of 3 or 4 numbers"); + return [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]; + } + if (info.dims > 1) { + if (!(v instanceof Array)) throw __err("value must be an array of " + info.dims + " numbers"); + var o = []; + for (var d = 0; d < info.dims; d++) o.push(d < v.length ? __num(v[d]) : (d === 2 ? 0 : __num(v[v.length - 1]))); + return o; + } + if (v instanceof Array) throw __err("value must be a number, not an array"); + return __num(v); + }; + P.__fromEngine = function (v, info) { + if (info.valueType === "TEXT_DOCUMENT") return TextDocument.__from(v); + if (info.valueType === "SHAPE") { + var s = new Shape(); + s.vertices = v.vertices; + s.inTangents = v.inTangents; + s.outTangents = v.outTangents; + s.closed = v.closed; + return s; + } + if (info.type === "color" && v instanceof Array) return v.slice(); + return v; + }; + P.valueAtTime = function (t, preExpression) { + var info = this.__info(); + var v = __get("value", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, time: __num(t, "time"), pre: !!preExpression }); + return this.__fromEngine(v, info); + }; + P.setValue = function (v) { + var info = this.__info(); + if (info.numKeys > 0) throw __err("Can not use setValue on \"" + info.name + "\": it has keyframes (use setValueAtTime or setValueAtKey)"); + if (info.hidden) throw __err("\"" + info.name + "\" can not be set"); + __call("prop.set", this.__ref({ value: this.__toEngine(v, info) })); + }; + P.setValueAtTime = function (t, v) { + var info = this.__info(); + t = __num(t, "time"); + __call("prop.addKey", this.__ref({ time: this.__layer.__layerTime(t), value: this.__toEngine(v, info, t) })); + }; + P.setValuesAtTimes = function (ts, vs) { + if (!(ts instanceof Array) || !(vs instanceof Array) || ts.length !== vs.length) throw __err("setValuesAtTimes: times and values must be arrays of the same length"); + for (var i = 0; i < ts.length; i++) this.setValueAtTime(ts[i], vs[i]); + }; + P.setValueAtKey = function (i, v) { + var k = this.__key(i), info = this.__info(); + __call("keys.set", this.__ref({ time: k.layerTime, value: this.__toEngine(v, info, k.time) })); + }; + P.addKey = function (t) { + t = __num(t, "time"); + __call("prop.addKey", this.__ref({ time: this.__layer.__layerTime(t) })); + return this.nearestKeyIndex(t); + }; + P.nearestKeyIndex = function (t) { + var k = this.__keys(), best = 0, d = Infinity; + if (!k.length) throw __err("\"" + this.name + "\" has no keyframes"); + for (var i = 0; i < k.length; i++) { + var e = Math.abs(k[i].time - t); + if (e < d) { d = e; best = i + 1; } + } + return best; + }; + P.keyTime = function (i) { return this.__key(i).time; }; + P.keyValue = function (i) { return this.__fromEngine(this.__key(i).value, this.__info()); }; + // Select key `i` (only) so the selection-based keyframe commands act on it. + P.__select = function (i) { + var k = this.__key(i); + __call("keys.select", { comp: this.__layer.__comp, keys: [{ layer: this.__layer.__id, prop: this.__uid, time: k.layerTime }] }); + return k; + }; + P.removeKey = function (i) { + this.__select(i); + __call("keys.delete", { comp: this.__layer.__comp }); + }; + P.keyInInterpolationType = function (i) { return KeyframeInterpolationType[this.__key(i).inInterp]; }; + P.keyOutInterpolationType = function (i) { return KeyframeInterpolationType[this.__key(i).outInterp]; }; + P.isInterpolationTypeValid = function (t) { return t === 6612 || t === 6613 || t === 6614; }; + P.setInterpolationTypeAtKey = function (i, inType, outType) { + if (outType === undefined) outType = inType; + if (!__interpKey[inType] || !__interpKey[outType]) throw __err("Bad KeyframeInterpolationType"); + this.__select(i); + __call("keys.interpolation", { comp: this.__layer.__comp, "in": __interpKey[inType], out: __interpKey[outType] }); + }; + P.keyInTemporalEase = function (i) { return this.__key(i).inEase.map(function (e) { return new KeyframeEase(e.speed, e.influence); }); }; + P.keyOutTemporalEase = function (i) { return this.__key(i).outEase.map(function (e) { return new KeyframeEase(e.speed, e.influence); }); }; + P.setTemporalEaseAtKey = function (i, inEase, outEase) { + if (outEase === undefined) outEase = inEase; + if (!(inEase instanceof Array) || !(outEase instanceof Array)) throw __err("setTemporalEaseAtKey: eases must be arrays of KeyframeEase"); + this.__select(i); + function sp(a) { return a.map(function (e) { return __num(e.speed, "speed"); }); } + function inf(a) { + return a.map(function (e) { + var x = __num(e.influence, "influence"); + if (x < 0.1 || x > 100) throw __err("influence must be between 0.1 and 100"); + return x; + }); + } + __call("keys.velocity", { comp: this.__layer.__comp, inSpeed: sp(inEase), inInfluence: inf(inEase), outSpeed: sp(outEase), outInfluence: inf(outEase) }); + }; + P.keyTemporalContinuous = function (i) { return !!this.__key(i).continuous; }; + P.setTemporalContinuousAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, continuous: !!b }); }; + P.keyTemporalAutoBezier = function (i) { return !!this.__key(i).autoBezier; }; + P.setTemporalAutoBezierAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, autoBezier: !!b }); }; + P.keyRoving = function (i) { return !!this.__key(i).roving; }; + P.setRovingAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, roving: !!b }); }; + P.keySpatialInTangent = function (i) { return this.__key(i).spatialIn.slice(0, this.__info().dims); }; + P.keySpatialOutTangent = function (i) { return this.__key(i).spatialOut.slice(0, this.__info().dims); }; + P.keySpatialContinuous = function (i) { return !!this.__key(i).spatialContinuous; }; + P.keySpatialAutoBezier = function (i) { return !!this.__key(i).spatialAutoBezier; }; + P.setSpatialTangentsAtKey = function (i, inT, outT) { + var k = this.__key(i); + __call("keys.setSpatialTangents", this.__ref({ time: k.layerTime, "in": inT, out: outT === undefined ? inT : outT })); + }; + P.setSpatialContinuousAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, spatial: b ? "continuousBezier" : "bezier" }); }; + P.setSpatialAutoBezierAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, spatial: b ? "autoBezier" : "bezier" }); }; + P.keySelected = function (i) { return !!this.__key(i).selected; }; + P.setSelectedAtKey = function (i, b) { + var k = this.__key(i); + if (b) { + __call("keys.select", { comp: this.__layer.__comp, add: true, keys: [{ layer: this.__layer.__id, prop: this.__uid, time: k.layerTime }] }); + } else { + var keep = [], all = this.__keys(); + for (var j = 0; j < all.length; j++) if (all[j].selected && j !== i - 1) keep.push({ layer: this.__layer.__id, prop: this.__uid, time: all[j].layerTime }); + __call("keys.select", { comp: this.__layer.__comp, keys: keep }); + } + }; + P.keyLabel = function () { return 0; }; + P.setLabelAtKey = function () {}; + P.addToMotionGraphicsTemplate = function () { return false; }; + P.canAddToMotionGraphicsTemplate = function () { return false; }; +})(); + +// Layer / composition markers (`layer.marker`, `comp.markerProperty`): keyframes are markers. +function MarkerProperty(comp, layer) { + this.__compId = comp; + this.__layerRef = layer; + this.__kind = "Property"; +} +MarkerProperty.prototype = { + constructor: MarkerProperty, + __p: function (o) { + o = o || {}; + o.comp = this.__compId; + if (this.__layerRef) o.layer = this.__layerRef.__id; + return o; + }, + __list: function () { return __get("markers", this.__p()); }, + get name() { return "Marker"; }, + get matchName() { return "ADBE Marker"; }, + get propertyValueType() { return PropertyValueType.MARKER; }, + get propertyType() { return PropertyType.PROPERTY; }, + get numKeys() { return this.__list().length; }, + get value() { return null; }, + get canSetExpression() { return false; }, + get canVaryOverTime() { return true; }, + get isTimeVarying() { return this.numKeys > 0; }, + get parentProperty() { return this.__layerRef || null; }, + __at: function (i) { + var l = this.__list(); + if (i < 1 || i > l.length) throw __err("marker index " + i + " is out of range 1.." + l.length); + return l[i - 1]; + }, + keyTime: function (i) { return this.__at(i).time; }, + keyValue: function (i) { + var m = this.__at(i); + var v = new MarkerValue(m.comment, m.chapter, m.url, m.frameTarget, m.cuePointName); + v.duration = m.duration; + v.eventCuePoint = m.eventCuePoint; + v.label = m.label; + v.protectedRegion = m.protectedRegion; + v.__params = m.params || []; + return v; + }, + nearestKeyIndex: function (t) { + var l = this.__list(), best = 0, d = Infinity; + for (var i = 0; i < l.length; i++) if (Math.abs(l[i].time - t) < d) { d = Math.abs(l[i].time - t); best = i + 1; } + if (!best) throw __err("there are no markers"); + return best; + }, + setValueAtTime: function (t, mv) { + t = __num(t, "time"); + if (!(mv instanceof MarkerValue)) throw __err("value must be a MarkerValue"); + var p = this.__p({ time: t, duration: mv.duration || 0, comment: mv.comment, chapter: mv.chapter, url: mv.url, frameTarget: mv.frameTarget, protected: !!mv.protectedRegion, label: mv.label || 0 }); + p.cuePoint = mv.cuePointName ? { name: mv.cuePointName, navigation: !mv.eventCuePoint, params: mv.__params } : null; + var l = this.__list(), at = -1; + for (var i = 0; i < l.length; i++) if (Math.abs(l[i].time - t) < 1e-6) at = i; + if (at >= 0) p.index = at; + else p["new"] = true; + __call("markers.set", p); + }, + setValueAtKey: function (i, mv) { this.setValueAtTime(this.keyTime(i), mv); }, + removeKey: function (i) { this.__at(i); __call("markers.delete", this.__p({ index: i - 1 })); }, + toString: function () { return "[object Property]"; }, +}; + +// ------------------------------------------------------------------ layers + +function __layer(compId, id) { + if (id === null || id === undefined) return null; + var i = __get("layer", { comp: compId, layer: id }); + switch (i.kind) { + case "text": return new TextLayer(compId, id); + case "shape": return new ShapeLayer(compId, id); + case "camera": return new CameraLayer(compId, id); + case "light": return new LightLayer(compId, id); + default: return new AVLayer(compId, id); + } +} + +function Layer() {} +Layer.prototype = { + __init: function (comp, id, kind) { + this.__comp = comp; + this.__id = id; + this.__kind = kind; + return new Proxy(this, __attr); + }, + __info: function () { return __get("layer", { comp: this.__comp, layer: this.__id }); }, + __p: function (o) { + o = o || {}; + o.comp = this.__comp; + o.layers = [this.__id]; + return o; + }, + __p1: function (o) { + o = o || {}; + o.comp = this.__comp; + o.layer = this.__id; + return o; + }, + __switch: function (sw, v) { __call("layer.setSwitch", this.__p({ "switch": sw, value: !!v })); }, + // Comp seconds → layer seconds (keyframe times are layer time). + __layerTime: function (t) { + var i = this.__info(); + return (t - i.startTime) * 100 / (i.stretch || 100); + }, + toString: function () { return "[object " + this.__kind + "]"; }, + get id() { return this.__id; }, + get name() { return this.__info().name; }, + set name(v) { __call("layer.rename", this.__p1({ name: String(v) })); }, + get index() { return this.__info().index; }, + get matchName() { return this.__info().matchName; }, + get containingComp() { return new CompItem(this.__comp); }, + get propertyDepth() { return 0; }, + get propertyType() { return PropertyType.INDEXED_GROUP; }, + get parentProperty() { return null; }, + get isModified() { return false; }, + get elided() { return false; }, + get isEffect() { return false; }, + get isMask() { return false; }, + get canSetEnabled() { return true; }, + get enabled() { return this.__info().enabled; }, + set enabled(v) { this.__switch("video", v); }, + get active() { return this.__info().active; }, + get solo() { return this.__info().solo; }, + set solo(v) { this.__switch("solo", v); }, + get locked() { return this.__info().locked; }, + set locked(v) { this.__switch("lock", v); }, + get shy() { return this.__info().shy; }, + set shy(v) { this.__switch("shy", v); }, + get audioEnabled() { return this.__info().audioEnabled; }, + set audioEnabled(v) { this.__switch("audio", v); }, + get hasVideo() { return this.__info().hasVideo; }, + get hasAudio() { return this.__info().hasAudio; }, + get audioActive() { var i = this.__info(); return i.hasAudio && i.audioEnabled; }, + get nullLayer() { return this.__info().nullLayer; }, + get isNameSet() { return true; }, + get inPoint() { return this.__info().inPoint; }, + set inPoint(v) { __call("layer.timing", this.__p({ "in": __num(v, "inPoint") })); }, + get outPoint() { return this.__info().outPoint; }, + set outPoint(v) { __call("layer.timing", this.__p({ out: __num(v, "outPoint") })); }, + get startTime() { return this.__info().startTime; }, + set startTime(v) { __call("layer.timing", this.__p({ start: __num(v, "startTime") })); }, + get stretch() { return this.__info().stretch; }, + set stretch(v) { __call("layer.timeStretch", this.__p({ percent: __num(v, "stretch") })); }, + get time() { return this.__info().time; }, + get label() { return this.__info().label; }, + set label(v) { + var names = __get("labels"); + var n = names[__num(v, "label")]; + if (n === undefined) throw __err("label must be 0-16"); + __call("edit.label", this.__p({ label: n })); + }, + get comment() { return this.__info().comment; }, + get selected() { return this.__info().selected; }, + set selected(v) { + var sel = this.containingComp.__info().selectedLayers; + var on = sel.indexOf(this.__id) >= 0; + if (!!v !== on) __call("layer.select", { comp: this.__comp, layers: [this.__id], toggle: true }); + }, + get selectedProperties() { + var c = this.containingComp.__info().selectedProperties, out = []; + for (var i = 0; i < c.length; i++) if (c[i][0] === this.__id) out.push(__propOf(this, __get("node", { comp: this.__comp, layer: this.__id, uid: c[i][1] }))); + return out; + }, + get parent() { var p = this.__info().parent; return p === null ? null : __layer(this.__comp, p); }, + set parent(v) { __call("layer.setParent", this.__p({ parent: v ? v.__id : null })); }, + setParentWithJump: function (v) { __call("layer.setParent", this.__p({ parent: v ? v.__id : null, compensate: false })); }, + get autoOrient() { return AutoOrientType[{ Off: "NO_AUTO_ORIENT", AlongPath: "ALONG_PATH", TowardsCamera: "CHARACTERS_TOWARD_CAMERA", TowardsPointOfInterest: "CAMERA_OR_POINT_OF_INTEREST" }[this.__info().autoOrient]]; }, + set autoOrient(v) { + var m = { 4212: "off", 4213: "alongPath", 4214: "towardsPointOfInterest", 4215: "towardsCamera" }[v]; + if (!m) throw __err("Bad AutoOrientType value " + v); + __call("layer.autoOrient", this.__p({ mode: m })); + }, + get marker() { return new MarkerProperty(this.__comp, this); }, + get numProperties() { return __get("node", { comp: this.__comp, layer: this.__id, uid: 0 }).numProperties; }, + property: function (key) { + if (key === undefined || key === null) throw __err("property(): missing name or index"); + if (key === "Marker" || key === "ADBE Marker" || key === "marker") return this.marker; + var k = typeof key === "number" ? key : String(key); + return __propOf(this, __get("child", { comp: this.__comp, layer: this.__id, uid: 0, key: k })); + }, + propertyGroup: function () { return null; }, + canAddProperty: function () { return false; }, + addProperty: function (n) { throw __err("Can not add \"" + n + "\" to a layer: use a property group (Effects, Masks, Contents)"); }, + remove: function () { __call("edit.clear", this.__p()); }, + duplicate: function () { + var before = this.containingComp.__info().layers; + __call("edit.duplicate", this.__p()); + var after = this.containingComp.__info().layers; + for (var i = 0; i < after.length; i++) if (before.indexOf(after[i]) < 0) return __layer(this.__comp, after[i]); + return null; + }, + copyToComp: function (comp) { + var c = __call("edit.copy", this.__p()); + var before = comp.__info().layers; + __call("comp.open", { comp: comp.__id }); + __call("edit.paste", { comp: comp.__id }); + return c; + }, + moveToBeginning: function () { __call("layer.arrange", this.__p({ to: "front" })); }, + moveToEnd: function () { __call("layer.arrange", this.__p({ to: "back" })); }, + moveBefore: function (other) { + var me = this.index, to = other.index; + __call("layer.arrange", this.__p({ to: "index", index: me < to ? to - 1 : to })); + }, + moveAfter: function (other) { + var me = this.index, to = other.index; + __call("layer.arrange", this.__p({ to: "index", index: me < to ? to : to + 1 })); + }, + activeAtTime: function (t) { var i = this.__info(); return i.enabled && t >= i.inPoint && t < i.outPoint; }, + applyPreset: function (f) { __call("anim.applyPreset", this.__p({ path: __str(f) })); }, +}; + +function AVLayer(comp, id) { return this.__init(comp, id, "AVLayer"); } +AVLayer.prototype = Object.create(Layer.prototype); +AVLayer.prototype.constructor = AVLayer; +(function () { + var P = AVLayer.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + def("source", function () { return __item(this.__info().source); }); + def("width", function () { return this.__info().width; }); + def("height", function () { return this.__info().height; }); + def("threeDLayer", function () { return this.__info().threeDLayer; }, function (v) { this.__switch("threeD", v); }); + def("adjustmentLayer", function () { return this.__info().adjustmentLayer; }, function (v) { this.__switch("adjustment", v); }); + def("guideLayer", function () { return this.__info().guideLayer; }, function (v) { this.__switch("guide", v); }); + def("motionBlur", function () { return this.__info().motionBlur; }, function (v) { this.__switch("motionBlur", v); }); + def("effectsActive", function () { return this.__info().effectsActive; }, function (v) { this.__switch("fx", v); }); + def("collapseTransformation", function () { return this.__info().collapse; }, function (v) { this.__switch("collapse", v); }); + def("preserveTransparency", function () { return this.__info().preserveTransparency; }, function (v) { this.__switch("preserveTransparency", v); }); + def("quality", function () { return LayerQuality[this.__info().quality.toUpperCase()]; }, function (v) { + var q = __enumKey(LayerQuality, v); + if (!q) throw __err("Bad LayerQuality value " + v); + __call("layer.quality", this.__p({ quality: q.toLowerCase() })); + }); + def("samplingQuality", function () { return LayerSamplingQuality[this.__info().samplingQuality.toUpperCase()]; }, function (v) { + var q = __enumKey(LayerSamplingQuality, v); + if (!q) throw __err("Bad LayerSamplingQuality value " + v); + __call("layer.sampling", this.__p({ sampling: q.toLowerCase() })); + }); + def("frameBlendingType", function () { return { Off: 4012, FrameMix: 4013, PixelMotion: 4014 }[this.__info().frameBlending]; }, function (v) { + var m = { 4012: "off", 4013: "frameMix", 4014: "pixelMotion" }[v]; + if (!m) throw __err("Bad FrameBlendingType value " + v); + __call("layer.frameBlending", this.__p({ mode: m })); + }); + def("frameBlending", function () { return this.__info().frameBlending !== "Off"; }); + def("blendingMode", function () { return __blendNames[this.__info().blendingMode]; }, function (v) { + __call("layer.setBlendMode", this.__p({ mode: __blendLabel(v) })); + }); + def("trackMatteType", function () { + var m = this.__info().trackMatte; + return m ? __matteEnum[m.kind] : TrackMatteType.NO_TRACK_MATTE; + }, function (v) { + if (v === TrackMatteType.NO_TRACK_MATTE) { __call("layer.trackMatte", this.__p({ op: "none" })); return; } + var k = __matteKind[v]; + if (!k) throw __err("Bad TrackMatteType value " + v); + var m = this.__info().trackMatte; + if (m) __call("layer.setTrackMatte", this.__p1({ matte: m.layer, kind: k })); + else __call("layer.trackMatte", this.__p({ op: k })); + }); + def("trackMatteLayer", function () { var m = this.__info().trackMatte; return m ? __layer(this.__comp, m.layer) : null; }); + def("hasTrackMatte", function () { return !!this.__info().trackMatte; }); + def("isTrackMatte", function () { return this.__info().isTrackMatte; }); + def("timeRemapEnabled", function () { return this.__info().timeRemapEnabled; }, function (v) { + __call("layer.enableTimeRemap", this.__p({ value: !!v })); + }); + def("canSetTimeRemapEnabled", function () { var s = this.source; return !!s && !(s instanceof FootageItem && s.mainSource.isStill); }); + def("canSetCollapseTransformation", function () { return true; }); + def("isNameFromSource", function () { var s = this.source; return !!s && s.name === this.name; }); + def("environmentLayer", function () { return false; }); + P.setTrackMatte = function (layer, type) { + if (!layer || type === TrackMatteType.NO_TRACK_MATTE) { this.removeTrackMatte(); return; } + var k = __matteKind[type]; + if (!k) throw __err("Bad TrackMatteType value " + type); + __call("layer.setTrackMatte", this.__p1({ matte: layer.__id, kind: k })); + }; + P.removeTrackMatte = function () { __call("layer.setTrackMatte", this.__p1({ matte: null })); }; + P.sourceRectAtTime = function (t, extents) { + return __get("sourceRect", { comp: this.__comp, layer: this.__id, time: __num(t, "time"), extents: !!extents }); + }; + P.sourcePointToComp = function (p) { return p; }; + P.compPointToSource = function (p) { return p; }; + P.replaceSource = function () { throw __err("replaceSource is not supported yet"); }; + P.openInViewer = function () { __call("layer.openLayer", this.__p1()); return null; }; +})(); + +function TextLayer(comp, id) { return this.__init(comp, id, "TextLayer"); } +TextLayer.prototype = Object.create(AVLayer.prototype); +TextLayer.prototype.constructor = TextLayer; +Object.defineProperty(TextLayer.prototype, "source", { get: function () { return null; } }); + +function ShapeLayer(comp, id) { return this.__init(comp, id, "ShapeLayer"); } +ShapeLayer.prototype = Object.create(AVLayer.prototype); +ShapeLayer.prototype.constructor = ShapeLayer; +Object.defineProperty(ShapeLayer.prototype, "source", { get: function () { return null; } }); + +function CameraLayer(comp, id) { return this.__init(comp, id, "CameraLayer"); } +CameraLayer.prototype = Object.create(Layer.prototype); +CameraLayer.prototype.constructor = CameraLayer; + +function LightLayer(comp, id) { return this.__init(comp, id, "LightLayer"); } +LightLayer.prototype = Object.create(Layer.prototype); +LightLayer.prototype.constructor = LightLayer; +Object.defineProperty(LightLayer.prototype, "lightType", { + get: function () { return LightType[this.__info().lightType.toUpperCase()]; }, + set: function (v) { + var k = __enumKey(LightType, v); + if (!k) throw __err("Bad LightType value " + v); + __call("layer.lightSettings", this.__p1({ kind: k.charAt(0) + k.substring(1).toLowerCase() })); + }, +}); + +// ------------------------------------------------------------------ collections + +function LayerCollection(comp) { + this.__comp = comp; + return new Proxy(this, __coll); +} +LayerCollection.prototype = { + constructor: LayerCollection, + __ids: function () { return __get("item", { id: this.__comp }).layers; }, + get length() { return this.__ids().length; }, + __at: function (i) { var ids = this.__ids(); return i >= 1 && i <= ids.length ? __layer(this.__comp, ids[i - 1]) : undefined; }, + __new: function (r) { return __layer(this.__comp, r.layer); }, + __dur: function (layer, duration) { + if (duration !== undefined && duration !== null) { + var i = layer.__info(); + __call("layer.timing", { comp: this.__comp, layers: [layer.__id], out: i.inPoint + __num(duration, "duration") }); + } + return layer; + }, + add: function (item, duration) { + if (!item || item.id === undefined) throw __err("add(): expected an item"); + var l = this.__new(__call("layer.addItem", { comp: this.__comp, item: item.id, time: 0 })); + return this.__dur(l, duration); + }, + addSolid: function (color, name, width, height, pixelAspect, duration) { + var p = { comp: this.__comp, color: [color[0], color[1], color[2]] }; + if (name !== undefined) p.name = String(name); + if (width !== undefined) p.width = Math.round(__num(width, "width")); + if (height !== undefined) p.height = Math.round(__num(height, "height")); + return this.__dur(this.__new(__call("layer.newSolid", p)), duration); + }, + addNull: function (duration) { return this.__dur(this.__new(__call("layer.newNull", { comp: this.__comp })), duration); }, + addText: function (text) { + var t = text instanceof TextDocument ? text : new TextDocument(text === undefined ? "" : String(text)); + var p = t.__attrs(); + p.comp = this.__comp; + if (p.text === undefined) p.text = t.text; + return this.__new(__call("layer.newText", p)); + }, + addBoxText: function (size, text) { + var c = __get("item", { id: this.__comp }); + var w = size[0], h = size[1]; + var t = text instanceof TextDocument ? text : new TextDocument(text === undefined ? "" : String(text)); + var p = t.__attrs(); + p.comp = this.__comp; + if (p.text === undefined) p.text = t.text; + p.box = [(c.width - w) / 2, (c.height - h) / 2, w, h]; + return this.__new(__call("layer.newText", p)); + }, + addShape: function () { return this.__new(__call("layer.newShape", { comp: this.__comp, kind: "none" })); }, + addCamera: function (name, center) { + var p = { comp: this.__comp, name: String(name) }; + if (center) p.poi = [center[0], center[1], 0]; + return this.__new(__call("layer.newCamera", p)); + }, + addLight: function (name, center) { + var p = { comp: this.__comp, name: String(name), kind: "Point" }; + if (center) p.poi = [center[0], center[1], 0]; + return this.__new(__call("layer.newLight", p)); + }, + precompose: function (indices, name, moveAllAttributes) { + var ids = this.__ids(), sel = []; + for (var i = 0; i < indices.length; i++) sel.push(ids[indices[i] - 1]); + var r = __call("layer.precompose", { comp: this.__comp, layers: sel, name: String(name), mode: moveAllAttributes === false ? "leave" : "move" }); + return new CompItem(r.comp); + }, + byName: function (name) { + var ids = this.__ids(); + for (var i = 0; i < ids.length; i++) { + var l = __get("layer", { comp: this.__comp, layer: ids[i] }); + if (l.name === name) return __layer(this.__comp, ids[i]); + } + return null; + }, + toString: function () { return "[object LayerCollection]"; }, +}; + +function ItemCollection(folder) { + this.__folder = folder; + return new Proxy(this, __coll); +} +ItemCollection.prototype = { + constructor: ItemCollection, + __ids: function () { + if (this.__folder === 0) return __get("project").items; + return __get("item", { id: this.__folder }).children; + }, + get length() { return this.__ids().length; }, + __at: function (i) { var ids = this.__ids(); return i >= 1 && i <= ids.length ? __item(ids[i - 1]) : undefined; }, + __place: function (id) { + if (this.__folder) __call("project.move", { items: [id], folder: this.__folder }); + return __item(id); + }, + addComp: function (name, width, height, pixelAspect, duration, frameRate) { + var r = __call("comp.new", { + name: String(name), width: Math.round(__num(width, "width")), height: Math.round(__num(height, "height")), + pixelAspect: __num(pixelAspect, "pixelAspect"), duration: __num(duration, "duration"), frameRate: __num(frameRate, "frameRate"), open: false, + }); + return this.__place(r.comp); + }, + addFolder: function (name) { + var r = __call("project.newFolder", { name: String(name) }); + var id = r && typeof r === "object" ? (r.folder || r.item || r.id) : r; + return this.__place(id); + }, + toString: function () { return "[object ItemCollection]"; }, +}; + +// ------------------------------------------------------------------ items + +function __item(id) { + if (id === null || id === undefined) return null; + if (id === 0) return new FolderItem(0); + var i = __get("item", { id: id }); + switch (i.kind) { + case "comp": return new CompItem(id); + case "folder": return new FolderItem(id); + default: return new FootageItem(id); + } +} + +function Item() {} +Item.prototype = { + __info: function () { return __get("item", { id: this.__id }); }, + toString: function () { return "[object " + this.__kind + "]"; }, + get id() { return this.__id; }, + get name() { return this.__info().name; }, + set name(v) { __call("project.rename", { item: this.__id, name: String(v) }); }, + get typeName() { return this.__info().typeName; }, + get comment() { return this.__info().comment; }, + set comment(v) { __call("project.setComment", { items: [this.__id], comment: String(v) }); }, + get label() { return this.__info().label; }, + set label(v) { __call("project.setLabel", { items: [this.__id], label: __num(v, "label") }); }, + get parentFolder() { return __item(this.__info().parent); }, + set parentFolder(f) { __call("project.move", { items: [this.__id], folder: f && f.__id ? f.__id : null }); }, + get selected() { return this.__info().selected; }, + set selected(v) { if (v) __call("project.select", { items: [this.__id], add: true }); }, + get usedIn() { return this.__info().usedIn.map(function (c) { return new CompItem(c); }); }, + remove: function () { __call("project.delete", { items: [this.__id] }); }, +}; + +function AVItem() {} +AVItem.prototype = Object.create(Item.prototype); +(function () { + function def(name, get, set) { Object.defineProperty(AVItem.prototype, name, { get: get, set: set }); } + def("width", function () { return this.__info().width; }, function (v) { this.__settings({ width: Math.round(__num(v, "width")) }); }); + def("height", function () { return this.__info().height; }, function (v) { this.__settings({ height: Math.round(__num(v, "height")) }); }); + def("pixelAspect", function () { return this.__info().pixelAspect; }, function (v) { this.__settings({ pixelAspect: __num(v) }); }); + def("duration", function () { return this.__info().duration; }, function (v) { this.__settings({ duration: __num(v, "duration") }); }); + def("frameRate", function () { return this.__info().frameRate; }, function (v) { this.__settings({ frameRate: __num(v, "frameRate") }); }); + def("frameDuration", function () { var f = this.__info().frameRate; return f ? 1 / f : 0; }, function (v) { this.__settings({ frameRate: 1 / __num(v) }); }); + def("hasVideo", function () { return this.__info().hasVideo; }); + def("hasAudio", function () { return this.__info().hasAudio; }); + def("footageMissing", function () { return !!this.__info().footageMissing; }); + def("useProxy", function () { return false; }); + def("proxySource", function () { return null; }); + def("isMediaReplacementCompatible", function () { return false; }); + def("time", function () { return 0; }); +})(); +AVItem.prototype.__settings = function () { throw __err("can't change this item's settings from a script"); }; + +function CompItem(id) { + this.__id = id; + this.__kind = "CompItem"; +} +CompItem.prototype = Object.create(AVItem.prototype); +CompItem.prototype.constructor = CompItem; +(function () { + var P = CompItem.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + P.__settings = function (o) { o.comp = this.__id; __call("comp.settings", o); }; + def("typeName", function () { return "Composition"; }); + def("layers", function () { return new LayerCollection(this.__id); }); + def("numLayers", function () { return this.__info().layers.length; }); + def("selectedLayers", function () { var c = this.__id; return this.__info().selectedLayers.map(function (l) { return __layer(c, l); }); }); + def("selectedProperties", function () { + var c = this.__id, out = []; + this.__info().selectedProperties.forEach(function (lp) { + var L = __layer(c, lp[0]); + out.push(__propOf(L, __get("node", { comp: c, layer: lp[0], uid: lp[1] }))); + }); + return out; + }); + def("activeCamera", function () { + var ids = this.__info().layers; + for (var i = 0; i < ids.length; i++) { + var l = __get("layer", { comp: this.__id, layer: ids[i] }); + if (l.kind === "camera" && l.enabled && l.active) return __layer(this.__id, ids[i]); + } + return null; + }); + def("time", function () { return this.__info().time; }, function (v) { __call("time.set", { comp: this.__id, time: __num(v, "time") }); }); + def("workAreaStart", function () { return this.__info().workAreaStart; }, function (v) { + var i = this.__info(); + __call("comp.workArea", { comp: this.__id, start: __num(v), end: __num(v) + i.workAreaDuration }); + }); + def("workAreaDuration", function () { return this.__info().workAreaDuration; }, function (v) { + var i = this.__info(); + __call("comp.workArea", { comp: this.__id, start: i.workAreaStart, end: i.workAreaStart + __num(v) }); + }); + def("bgColor", function () { return this.__info().bgColor; }, function (v) { this.__settings({ background: [v[0], v[1], v[2]] }); }); + def("displayStartTime", function () { return this.__info().displayStartTime; }, function (v) { this.__settings({ startTime: __num(v) }); }); + def("hideShyLayers", function () { return this.__info().hideShyLayers; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "hideShy", value: !!v }); }); + def("motionBlur", function () { return this.__info().motionBlur; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "motionBlur", value: !!v }); }); + def("frameBlending", function () { return this.__info().frameBlending; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "frameBlending", value: !!v }); }); + def("draft3d", function () { return this.__info().draft3d; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "draft3d", value: !!v }); }); + def("shutterAngle", function () { return this.__info().shutterAngle; }, function (v) { this.__settings({ shutterAngle: __num(v) }); }); + def("shutterPhase", function () { return this.__info().shutterPhase; }, function (v) { this.__settings({ shutterPhase: __num(v) }); }); + def("motionBlurSamplesPerFrame", function () { return this.__info().motionBlurSamplesPerFrame; }, function (v) { this.__settings({ motionBlurSamples: __num(v) }); }); + def("renderer", function () { return this.__info().renderer; }, function (v) { + this.__settings({ renderer: String(v).toLowerCase().indexOf("advanced") >= 0 ? "advanced3D" : "classic3D" }); + }); + def("renderers", function () { return ["ADBE Classic 3D", "ADBE Advanced 3d"]; }); + def("markerProperty", function () { return new MarkerProperty(this.__id, null); }); + def("hasVideo", function () { return true; }); + P.layer = function (a, b) { + var ids = this.__info().layers; + if (typeof a === "number") { + if (b !== undefined) { + // layer(otherLayer, relIndex) + throw __err("layer(index, relIndex) is not supported"); + } + if (a < 1 || a > ids.length || Math.floor(a) !== a) throw __err("layer index " + a + " is out of range 1.." + ids.length); + return __layer(this.__id, ids[a - 1]); + } + if (a && a.__id !== undefined && typeof b === "number") { + var j = a.index + b; + return j >= 1 && j <= ids.length ? __layer(this.__id, ids[j - 1]) : null; + } + return new LayerCollection(this.__id).byName(String(a)); + }; + P.openInViewer = function () { __call("comp.open", { comp: this.__id }); return null; }; + P.duplicate = function () { return __item(__call("project.duplicate", { items: [this.__id] }).items[0]); }; + P.exportAsMotionGraphicsTemplate = function () { return false; }; +})(); + +function FootageItem(id) { + this.__id = id; + this.__kind = "FootageItem"; +} +FootageItem.prototype = Object.create(AVItem.prototype); +FootageItem.prototype.constructor = FootageItem; +Object.defineProperty(FootageItem.prototype, "file", { + get: function () { var f = this.__info().file; return f ? new File(f) : null; }, +}); +Object.defineProperty(FootageItem.prototype, "mainSource", { + get: function () { + var i = this.__info(); + if (i.kind === "solid") return { isStill: true, color: i.color, hasAlpha: false, toString: function () { return "[object SolidSource]"; } }; + return { isStill: !!i.isStill, file: i.file ? new File(i.file) : null, missingFootagePath: i.footageMissing ? i.file : "", loop: i.loop, hasAlpha: false, toString: function () { return "[object FileSource]"; } }; + }, +}); +FootageItem.prototype.replace = function (f) { __call("file.replaceFootage", { item: this.__id, path: __str(f) }); }; +FootageItem.prototype.replaceWithSolid = function (color, name, w, h) { + __call("file.replaceWithSolid", { item: this.__id, color: [color[0], color[1], color[2]], width: w, height: h }); + if (name !== undefined) this.name = name; +}; +FootageItem.prototype.openInViewer = function () { return null; }; + +function FolderItem(id) { + this.__id = id; + this.__kind = "FolderItem"; +} +FolderItem.prototype = Object.create(Item.prototype); +FolderItem.prototype.constructor = FolderItem; +(function () { + var P = FolderItem.prototype; + P.__info = function () { + if (this.__id === 0) return { id: 0, name: "Root", typeName: "Folder", comment: "", label: 0, parent: null, selected: false, usedIn: [], children: [] }; + return __get("item", { id: this.__id }); + }; + Object.defineProperty(P, "items", { get: function () { return new ItemCollection(this.__id); } }); + Object.defineProperty(P, "numItems", { get: function () { return new ItemCollection(this.__id).length; } }); + Object.defineProperty(P, "parentFolder", { + get: function () { return this.__id === 0 ? null : __item(this.__info().parent); }, + set: function (f) { __call("project.move", { items: [this.__id], folder: f && f.__id ? f.__id : null }); }, + }); + P.item = function (i) { + var v = new ItemCollection(this.__id).__at(__num(i, "index")); + if (v === undefined) throw __err("item index " + i + " is out of range"); + return v; + }; +})(); + +// ------------------------------------------------------------------ render queue + +function OutputModule(rq, n) { + this.__rq = rq; + this.__n = n; +} +OutputModule.prototype = { + constructor: OutputModule, + __info: function () { return this.__rq.__info(); }, + get name() { var i = this.__info(); return this.__n === 1 ? i.outputModuleSummary : i.extraOutputs[this.__n - 2].summary; }, + get file() { + var i = this.__info(); + var p = this.__n === 1 ? i.outputPath : i.extraOutputs[this.__n - 2].outputPath; + return p ? new File(p) : null; + }, + set file(f) { __call("renderQueue.setOutput", { item: this.__rq.__id, module: this.__n, path: __str(f) }); }, + get templates() { return ["H.264", "ProRes", "PNG Sequence", "JPEG Sequence", "TIFF Sequence", "OpenEXR Sequence", "GIF"]; }, + applyTemplate: function (name) { + var n = String(name).toLowerCase(); + var f = n.indexOf("prores") >= 0 ? "prores" : n.indexOf("png") >= 0 || n.indexOf("lossless") >= 0 ? "png" : n.indexOf("jpeg") >= 0 || n.indexOf("jpg") >= 0 ? "jpeg" + : n.indexOf("tiff") >= 0 ? "tiff" : n.indexOf("exr") >= 0 ? "exr" : n.indexOf("gif") >= 0 ? "gif" : n.indexOf("264") >= 0 ? "h264" : null; + if (!f) throw __err("Unknown output module template \"" + name + "\""); + __call("renderQueue.setOutputModule", { item: this.__rq.__id, module: this.__n, format: f }); + }, + remove: function () { throw __err("removing output modules is not supported"); }, + toString: function () { return "[object OutputModule]"; }, +}; + +var __rqStatus = { "Unqueued": 3014, "Queued": 3015, "Needs Output": 3013, "Rendering": 3016, "Done": 3019, "User Stopped": 3017, "Failed": 3018 }; + +function RenderQueueItem(id) { this.__id = id; } +RenderQueueItem.prototype = { + constructor: RenderQueueItem, + __info: function () { + var l = __call("renderQueue.list", {}).items; + for (var i = 0; i < l.length; i++) if (l[i].id === this.__id) return l[i]; + throw __err("the render queue item no longer exists"); + }, + get comp() { return new CompItem(this.__info().comp); }, + get status() { + var s = this.__info().statusLabel; + for (var k in __rqStatus) if (s.indexOf(k) === 0) return __rqStatus[k]; + return RQItemStatus.QUEUED; + }, + get render() { return this.__info().render; }, + set render(v) { __call("renderQueue.setRender", { item: this.__id, render: !!v }); }, + get numOutputModules() { return this.__info().outputModules; }, + get outputModules() { + var n = this.numOutputModules, out = []; + for (var i = 1; i <= n; i++) out.push(new OutputModule(this, i)); + return out; + }, + outputModule: function (i) { + if (i < 1 || i > this.numOutputModules) throw __err("output module index " + i + " is out of range"); + return new OutputModule(this, i); + }, + get timeSpanStart() { var s = this.__info().settings; return s.start !== undefined ? s.start : 0; }, + set timeSpanStart(v) { __call("renderQueue.setRenderSettings", { item: this.__id, timeSpan: "custom", start: __num(v), end: __num(v) + this.timeSpanDuration }); }, + get timeSpanDuration() { var s = this.__info().settings; return s.end !== undefined && s.start !== undefined ? s.end - s.start : this.comp.duration; }, + set timeSpanDuration(v) { var s = this.timeSpanStart; __call("renderQueue.setRenderSettings", { item: this.__id, timeSpan: "custom", start: s, end: s + __num(v) }); }, + get elapsedSeconds() { return this.__info().render_time || 0; }, + get startTime() { var s = this.__info().started; return s ? new Date(s * 1000) : null; }, + get logType() { return LogType.ERRORS_ONLY; }, + set logType(v) {}, + get skipFrames() { return 0; }, + outputModuleAdd: function () { __call("render.addOutputModule", { item: this.__id }); }, + remove: function () { __call("renderQueue.remove", { item: this.__id }); }, + duplicate: function () { + var before = __call("renderQueue.list", {}).items.map(function (i) { return i.id; }); + __call("renderQueue.duplicate", { item: this.__id }); + var after = __call("renderQueue.list", {}).items; + for (var i = 0; i < after.length; i++) if (before.indexOf(after[i].id) < 0) return new RenderQueueItem(after[i].id); + return null; + }, + applyTemplate: function () {}, + getSettings: function () { return this.__info().settings; }, + setSettings: function (o) { __call("renderQueue.setRenderSettings", Object.assign({ item: this.__id }, o)); }, + toString: function () { return "[object RenderQueueItem]"; }, +}; + +function RQItemCollection() { return new Proxy(this, __coll); } +RQItemCollection.prototype = { + constructor: RQItemCollection, + __list: function () { return __call("renderQueue.list", {}).items; }, + get length() { return this.__list().length; }, + __at: function (i) { var l = this.__list(); return i >= 1 && i <= l.length ? new RenderQueueItem(l[i - 1].id) : undefined; }, + add: function (comp) { + if (!(comp instanceof CompItem)) throw __err("add(): expected a CompItem"); + __call("renderQueue.add", { comp: comp.__id }); + var l = this.__list(); + return new RenderQueueItem(l[l.length - 1].id); + }, + toString: function () { return "[object RQItemCollection]"; }, +}; + +function RenderQueue() {} +RenderQueue.prototype = { + constructor: RenderQueue, + get items() { return new RQItemCollection(); }, + get numItems() { return new RQItemCollection().length; }, + get rendering() { return !!__call("renderQueue.list", {}).rendering; }, + get canQueueInAME() { return false; }, + get queueNotify() { return false; }, + set queueNotify(v) {}, + item: function (i) { + var v = new RQItemCollection().__at(__num(i, "index")); + if (v === undefined) throw __err("render queue index " + i + " is out of range"); + return v; + }, + render: function () { __call("renderQueue.render", { wait: true }); }, + stopRendering: function () { __call("renderQueue.stop", {}); }, + pauseRendering: function () {}, + showWindow: function () {}, + queueInAME: function () { throw __err("Adobe Media Encoder is not available: render with app.project.renderQueue.render()"); }, + toString: function () { return "[object RenderQueue]"; }, +}; + +// ------------------------------------------------------------------ project and app + +function ImportOptions(file) { + if (!(this instanceof ImportOptions)) return new ImportOptions(file); + this.file = file || null; + this.importAs = ImportAsType.FOOTAGE; + this.sequence = false; + this.forceAlphabetical = false; + this.rangeStart = 0; + this.rangeEnd = 0; +} +ImportOptions.prototype.canImportAs = function (t) { return t === ImportAsType.FOOTAGE; }; + +function Project() {} +Project.prototype = { + constructor: Project, + __info: function () { return __get("project"); }, + get items() { return new ItemCollection(0); }, + get numItems() { return this.__info().items.length; }, + get rootFolder() { return new FolderItem(0); }, + get activeItem() { return __item(this.__info().active); }, + get selection() { return this.__info().selection.map(__item); }, + get file() { var p = this.__info().path; return p ? new File(p) : null; }, + get dirty() { return this.__info().dirty; }, + get renderQueue() { return new RenderQueue(); }, + get bitsPerChannel() { return this.__info().bitsPerChannel; }, + set bitsPerChannel(v) { __call("file.projectSettings", { bitDepth: __num(v) }); }, + get linearBlending() { return this.__info().linearBlending; }, + set linearBlending(v) { __call("file.projectSettings", { blendLinear: !!v }); }, + get linearizeWorkingSpace() { return this.__info().linearizeWorkingSpace; }, + set linearizeWorkingSpace(v) { __call("file.projectSettings", { linearize: !!v }); }, + get workingSpace() { return this.__info().workingSpace; }, + set workingSpace(v) { __call("file.projectSettings", { workingSpace: String(v) || "none" }); }, + get timeDisplayType() { return this.__info().timeDisplayFrames ? TimeDisplayType.FRAMES : TimeDisplayType.TIMECODE; }, + set timeDisplayType(v) { __call("file.projectSettings", { timeDisplay: v === TimeDisplayType.FRAMES ? "frames" : "timecode" }); }, + get expressionEngine() { return "javascript-1.0"; }, + set expressionEngine(v) {}, + item: function (i) { + var v = new ItemCollection(0).__at(__num(i, "index")); + if (v === undefined) throw __err("item index " + i + " is out of range 1.." + this.numItems); + return v; + }, + itemByID: function (id) { + try { return __item(__num(id)); } catch (e) { return null; } + }, + layerByID: function () { return null; }, + save: function (file) { + if (file) __call("file.saveAs", { path: __str(file) }); + else __call("file.save", {}); + return true; + }, + saveWithDialog: function () { if (this.file) return this.save(); return false; }, + close: function (opt) { + if (opt === CloseOptions.SAVE_CHANGES && this.file) this.save(); + __call("file.newProject", {}); + return true; + }, + importFile: function (opts) { + var f = opts instanceof ImportOptions ? opts.file : opts; + if (!f) throw __err("importFile(): no file"); + var r = __call("file.import", { paths: [__str(f)] }); + if (r.errors && r.errors.length) throw __err(r.errors.join("; ")); + return __item(r.items[0]); + }, + importFileWithDialog: function () { return null; }, + importPlaceholder: function (name, w, h, rate, dur) { + var r = __call("file.importPlaceholder", { name: String(name), width: w, height: h, frameRate: rate, duration: dur }); + var id = r && typeof r === "object" ? (r.item || r.id) : r; + return __item(id); + }, + consolidateFootage: function () { return __call("file.consolidateFootage", {}); }, + removeUnusedFootage: function () { return __call("file.removeUnusedFootage", {}); }, + reduceProject: function (items) { + __call("project.select", { items: items.map(function (i) { return i.__id; }) }); + return __call("file.reduceProject", {}); + }, + autoFixExpressions: function () {}, + showWindow: function () {}, + toString: function () { return "[object Project]"; }, +}; + +var __tasks = {}; +var app = { + get project() { return new Project(); }, + get version() { return __get("app").version; }, + get buildName() { return __get("app").buildName; }, + get buildNumber() { return 0; }, + get isWatchFolder() { return false; }, + get isRenderEngine() { return false; }, + get isoLanguage() { return "en_US"; }, + get language() { return 0; }, + get memoryInUse() { return 0; }, + get effects() { return __get("effects").map(function (e) { return { displayName: e.name, matchName: e.matchName, category: e.category, version: "1.0" }; }); }, + get activeViewer() { return null; }, + get availableGPUAccelTypes() { return []; }, + exitAfterLaunchAndEval: false, + exitCode: 0, + saveProjectOnCrash: true, + onError: null, + settings: { + __s: {}, + haveSetting: function (sec, key) { return (sec + "/" + key) in this.__s; }, + getSetting: function (sec, key) { return this.__s[sec + "/" + key] || ""; }, + saveSetting: function (sec, key, v) { this.__s[sec + "/" + key] = String(v); }, + }, + preferences: { + havePref: function () { return false; }, + getPrefAsString: function () { return ""; }, + getPrefAsLong: function () { return 0; }, + getPrefAsBool: function () { return false; }, + getPrefAsFloat: function () { return 0; }, + savePrefAsString: function () {}, + savePrefAsLong: function () {}, + savePrefAsBool: function () {}, + savePrefAsFloat: function () {}, + reload: function () {}, + }, + newProject: function () { __call("file.newProject", {}); return new Project(); }, + open: function (file) { + if (!file) return null; + __call("file.open", { path: __str(file) }); + return new Project(); + }, + quit: function () {}, + beginUndoGroup: function (name) { __undo("begin", name === undefined ? "Script" : String(name)); }, + endUndoGroup: function () { __undo("end", ""); }, + beginSuppressDialogs: function () {}, + endSuppressDialogs: function () {}, + purge: function () { __call("edit.purge", { what: "all" }); }, + findMenuCommandId: function (name) { return __get("menuCommandId", { name: String(name) }); }, + executeCommand: function (id) { + var cmd = typeof id === "number" ? __get("commandAt", { index: id }) : String(id); + if (!cmd) throw __err("Unknown command id " + id); + return __call(cmd, {}); + }, + scheduleTask: function (code, delay, repeat) { return __task("schedule", String(code), !!repeat); }, + cancelTask: function (id) { __task("cancel", id); }, + setMemoryUsageLimits: function () {}, + setMultiFrameRenderingConfig: function () {}, + setSavePreferencesOnQuit: function () {}, + activate: function () {}, + watchFolder: function () { throw __err("watch folders are not supported"); }, + pauseWatchFolder: function () {}, + endWatchFolder: function () {}, + parseSwatchFile: function () { throw __err("parseSwatchFile is not supported"); }, + // EffectCraft: run any engine command by id with parameters (see `command.list`). + run: function (id, params) { return __call(String(id), params || {}); }, + toString: function () { return "[object Application]"; }, +}; + +// The completion value of a script, as JSON text for the host. +function __result(v) { + if (v === undefined) return undefined; + if (v === null || typeof v === "number" || typeof v === "boolean" || typeof v === "string") return JSON.stringify(v); + if (v instanceof Layer || v instanceof Item || v instanceof PropertyBase || v instanceof Project || v instanceof TextDocument) { + var d = { type: v.toString() }; + try { d.name = v.name; } catch (e) {} + if (v instanceof TextDocument) d.text = v.text; + return JSON.stringify(d); + } + try { + var s = JSON.stringify(v); + return s === undefined ? JSON.stringify(String(v)) : s; + } catch (e) { + return JSON.stringify(String(v)); + } +} diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs new file mode 100644 index 000000000..61b495950 --- /dev/null +++ b/crates/script/src/runtime.rs @@ -0,0 +1,546 @@ +//! The boa runtime: native functions (`__exec`, `__query`, `__print`, `__undo`, `__task`, +//! `__file`), the session hand-off while a script runs, undo groups and error reporting. + +use std::cell::RefCell; +use std::mem::ManuallyDrop; +use std::sync::Arc; + +use boa_engine::{Context, JsNativeError, JsResult, JsString, JsValue, NativeFunction, Source, js_string}; +use effectcraft_engine::project::Project; +use effectcraft_engine::{ScriptRequest, Session}; +use serde_json::{Value as J, json}; + +const PRELUDE: &str = include_str!("prelude.js"); + +/// Engine commands that create layers: like After Effects, scripts add new layers at the top of +/// the stack (the UI inserts above the selection). +const CREATES_LAYER: &[&str] = + &["layer.newSolid", "layer.newNull", "layer.newText", "layer.newShape", "layer.newAdjustment", "layer.newCamera", "layer.newLight", "layer.addItem"]; + +/// What a script run produced. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Outcome { + /// The value of the script's last expression statement (JSON when it converts; model objects + /// as strings like `"[object CompItem]"`). + pub result: J, + /// `writeLn` / `$.writeln` / `alert` output, one entry per line. + pub output: Vec, + pub error: Option, +} + +/// An uncaught script error. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScriptError { + pub message: String, + pub line: Option, + pub column: Option, + pub file: String, +} + +impl Outcome { + pub fn ok(&self) -> bool { + self.error.is_none() + } + pub fn to_json(&self) -> J { + json!({ + "ok": self.ok(), + "result": self.result, + "output": self.output.join("\n"), + "error": self.error.as_ref().map(|e| json!({"message": e.message, "line": e.line, "column": e.column, "file": e.file})), + }) + } +} + +/// The session and per-run state while a script runs (the natives reach it here; nothing +/// borrowed ever enters the JS heap). +struct Active { + session: Session, + output: Vec, + /// Open undo groups: (name, project snapshot when the outermost group began). + groups: Vec<(String, Arc)>, + /// `app.scheduleTask` queue: (id, code, repeat). + tasks: Vec<(u32, String, bool)>, + next_task: u32, + name: String, +} + +thread_local! { + static ACTIVE: RefCell> = const { RefCell::new(None) }; + /// The Script Console's persistent context (never dropped, like the expression runtime: boa's + /// own thread-local heap may be torn down first at thread exit). + static CONSOLE: RefCell>> = const { RefCell::new(None) }; +} + +fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { + ACTIVE.with(|a| match a.try_borrow_mut() { + Ok(mut g) => match g.as_mut() { + Some(act) => f(act), + None => Err("no script is running".into()), + }, + Err(_) => Err("scripts can't run scripts".into()), + }) +} + +fn throw(msg: impl Into) -> boa_engine::JsError { + JsNativeError::error().with_message(msg.into()).into() +} + +fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { + Ok(args.get(i).cloned().unwrap_or_default().to_string(ctx)?.to_std_string_escaped()) +} + +fn arg_json(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { + let s = arg_string(args, i, ctx)?; + if s.is_empty() || s == "undefined" { + return Ok(json!({})); + } + serde_json::from_str(&s).map_err(|e| throw(format!("bad parameters: {e}"))) +} + +fn js_str(s: &str) -> JsValue { + JsValue::from(JsString::from(s)) +} + +/// `__exec(commandId, paramsJson)` → result JSON. Runs an engine command (undoable) with the +/// target comp made active for its duration. +fn native_exec(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let id = arg_string(args, 0, ctx)?; + let params = arg_json(args, 1, ctx)?; + if id == "script.run" || id == "file.runScript" { + return Err(throw("scripts can't run scripts")); + } + let r = with_active(|a| Ok(exec(&mut a.session, &id, params))); + match r { + Ok(Ok(v)) => Ok(js_str(&v.to_string())), + Ok(Err(e)) | Err(e) => Err(throw(e)), + } +} + +/// Run a command for a script: the `comp` it targets is active while it runs (commands act on +/// the active comp), then the user's active comp comes back. +pub(crate) fn exec(s: &mut Session, id: &str, params: J) -> Result { + use effectcraft_engine::project::ItemId; + let target = params.get("comp").and_then(J::as_u64).map(ItemId).filter(|c| s.project.comp(*c).is_some()); + let prev = s.state.active_comp; + if let Some(c) = target + && prev != Some(c) + { + s.state.active_comp = Some(c); + } + if CREATES_LAYER.contains(&id) { + s.state.selected_layers.clear(); + } + let r = s.execute(id, params); + if let Some(c) = target + && prev != Some(c) + && s.state.active_comp == Some(c) + && id != "comp.open" + { + match prev { + Some(p) if s.project.comp(p).is_some() => s.state.active_comp = Some(p), + _ => { + if !s.state.open_comps.contains(&c) { + s.state.open_comps.push(c); + } + } + } + } + r.map_err(|e| e.to_string()) +} + +/// `__query(kind, json)` → JSON. +fn native_query(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let kind = arg_string(args, 0, ctx)?; + let a = arg_json(args, 1, ctx)?; + let r = with_active(|act| match kind.as_str() { + "app" => Ok(json!({ + "version": format!("26.0 (EffectCraft {})", env!("CARGO_PKG_VERSION")), + "buildName": env!("CARGO_PKG_VERSION"), + "scriptName": act.name, + "allowFiles": act.session.prefs.scripting.allow_scripts_write_files, + "projectDir": project_dir(&act.session), + })), + _ => crate::model::query(&mut act.session, &kind, &a), + }); + match r { + Ok(v) => Ok(js_str(&v.to_string())), + Err(e) => Err(throw(e)), + } +} + +/// `__print(text)`: `writeLn`, `$.writeln`, `alert`. +fn native_print(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let text = arg_string(args, 0, ctx)?; + let _ = with_active(|a| { + a.output.extend(text.split('\n').map(str::to_string)); + Ok(()) + }); + Ok(JsValue::undefined()) +} + +/// `__undo("begin" | "end", name)`: After Effects' `app.beginUndoGroup` / `endUndoGroup`. Every +/// command run inside the outermost group folds into one undo step named after the group. +fn native_undo(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + let name = arg_string(args, 1, ctx)?; + with_active(|a| { + match op.as_str() { + "begin" => { + let snap = a.session.project.clone(); + a.groups.push((if name.is_empty() || name == "undefined" { "Script".into() } else { name }, snap)); + } + _ => { + if a.groups.len() == 1 { + let (name, snap) = a.groups.pop().unwrap_or_default(); + collapse(&mut a.session, &name, &snap); + } else { + a.groups.pop(); + } + } + } + Ok(()) + }) + .map_err(throw)?; + Ok(JsValue::undefined()) +} + +/// Fold the undo steps recorded since the project was `snap` into one step called `name`. +fn collapse(s: &mut Session, name: &str, snap: &Arc) { + let h = &mut s.history; + let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { + // Nothing was recorded (or history was reset by New / Open Project). + return; + }; + let first = h.undo[start].1.clone(); + h.undo.truncate(start); + h.undo.push((name.to_string(), first)); + h.merge_key = None; +} + +/// `__task("schedule", code, repeat)` → id; `__task("cancel", id)`. +fn native_task(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + if op == "cancel" { + let id = args.get(1).cloned().unwrap_or_default().to_number(ctx)? as u32; + let _ = with_active(|a| { + a.tasks.retain(|t| t.0 != id); + Ok(()) + }); + return Ok(JsValue::undefined()); + } + let code = arg_string(args, 1, ctx)?; + let repeat = args.get(2).cloned().unwrap_or_default().to_boolean(); + let id = with_active(|a| { + a.next_task += 1; + let id = a.next_task; + a.tasks.push((id, code, repeat)); + Ok(id) + }) + .map_err(throw)?; + Ok(JsValue::from(id)) +} + +fn project_dir(s: &Session) -> Option { + let p = s.path.as_deref()?; + std::path::Path::new(p).parent().map(|d| d.to_string_lossy().to_string()) +} + +/// Lexically normalized absolute path (`a/./b/../c` → `a/c`). +fn normalize(p: &str) -> std::path::PathBuf { + use std::path::{Component, Path, PathBuf}; + let path = Path::new(p); + let abs = if path.is_absolute() { path.to_path_buf() } else { std::env::current_dir().unwrap_or_default().join(path) }; + let mut out = PathBuf::new(); + for c in abs.components() { + match c { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + c => out.push(c), + } + } + // Resolve symlinks where the file (or its folder) exists, so links can't escape the project. + std::fs::canonicalize(&out) + .or_else(|_| out.parent().map(std::fs::canonicalize).unwrap_or(Ok(out.clone())).map(|d| d.join(out.file_name().unwrap_or_default()))) + .unwrap_or(out) +} + +const GATE: &str = "Preferences ▸ Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network"; + +/// The file-access gate: reads under the project's folder are always allowed; other reads, all +/// writes and listing other folders need the Allow Scripts to Write Files preference. +fn check_access(s: &Session, path: &str, write: bool) -> Result { + let p = normalize(path); + if s.prefs.scripting.allow_scripts_write_files { + return Ok(p); + } + if write { + return Err(format!("Unable to write {path}: scripts can't write files unless {GATE} is on")); + } + if let Some(dir) = project_dir(s) { + let d = normalize(&dir); + if p.starts_with(&d) { + return Ok(p); + } + } + Err(format!("Unable to access {path}: scripts can only read files in the project's folder unless {GATE} is on")) +} + +/// `__file(op, path, data)`: `read` → text, `write`/`append` → bytes, `exists` → bool, `list` → +/// names, `mkdir`, `remove`, `isDir`. +fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + let path = arg_string(args, 1, ctx)?; + let data = if args.len() > 2 { arg_string(args, 2, ctx)? } else { String::new() }; + let r: Result = with_active(|a| { + let s = &a.session; + match op.as_str() { + "access" => Ok(json!(check_access(s, &path, false)?.exists())), + "exists" | "isDir" => { + // Unreadable locations report "missing" instead of failing. + let Ok(p) = check_access(s, &path, false) else { return Ok(json!(false)) }; + Ok(json!(if op == "exists" { p.exists() } else { p.is_dir() })) + } + "read" => { + let p = check_access(s, &path, false)?; + let bytes = s.services.read_file(&p.to_string_lossy()).map_err(|e| format!("Unable to read {path}: {e}"))?; + Ok(json!(String::from_utf8_lossy(&bytes))) + } + "write" | "append" => { + let p = check_access(s, &path, true)?; + let ps = p.to_string_lossy().to_string(); + let mut bytes = if op == "append" { s.services.read_file(&ps).unwrap_or_default() } else { vec![] }; + bytes.extend_from_slice(data.as_bytes()); + s.services.write_file(&ps, &bytes).map_err(|e| format!("Unable to write {path}: {e}"))?; + Ok(json!(bytes.len())) + } + "list" => { + let p = check_access(s, &path, false)?; + let mut names: Vec = std::fs::read_dir(&p) + .map_err(|e| format!("Unable to list {path}: {e}"))? + .filter_map(|e| e.ok()) + .map(|e| e.path().to_string_lossy().to_string()) + .collect(); + names.sort(); + Ok(json!(names)) + } + "mkdir" => { + let p = check_access(s, &path, true)?; + std::fs::create_dir_all(&p).map_err(|e| format!("Unable to create {path}: {e}"))?; + Ok(json!(true)) + } + "remove" => { + let p = check_access(s, &path, true)?; + let r = if p.is_dir() { std::fs::remove_dir(&p) } else { std::fs::remove_file(&p) }; + Ok(json!(r.is_ok())) + } + "network" => Err(if s.prefs.scripting.allow_scripts_write_files { + "network access from scripts is not supported in EffectCraft".to_string() + } else { + format!("scripts can't access the network unless {GATE} is on") + }), + _ => Err(format!("unknown file operation `{op}`")), + } + }); + match r { + Ok(v) => Ok(js_str(&v.to_string())), + Err(e) => Err(throw(e)), + } +} + +fn new_context() -> Result { + let mut ctx = Context::default(); + let limits = ctx.runtime_limits_mut(); + limits.set_loop_iteration_limit(200_000_000); + limits.set_recursion_limit(2000); + let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 6] = [ + (js_string!("__exec"), 2, native_exec), + (js_string!("__query"), 2, native_query), + (js_string!("__print"), 1, native_print), + (js_string!("__undo"), 2, native_undo), + (js_string!("__task"), 3, native_task), + (js_string!("__file"), 3, native_file), + ]; + for (name, len, f) in natives { + ctx.register_global_callable(name, len, NativeFunction::from_fn_ptr(f)).map_err(|e| e.to_string())?; + } + ctx.eval(Source::from_bytes(PRELUDE)).map_err(|e| format!("script prelude: {e}"))?; + Ok(ctx) +} + +/// Turn a boa error into a message with line and column (`code` is the user's script). +fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { + // Thrown values: `throw new Error("x")`, `throw "x"`, native errors. + let display = e.to_string(); + let message = match e.as_opaque() { + Some(v) => { + let o = v.as_object(); + let msg = o.as_ref().and_then(|o| o.get(js_string!("message"), ctx).ok()).filter(|m| !m.is_undefined()); + let name = o.as_ref().and_then(|o| o.get(js_string!("name"), ctx).ok()).filter(|m| !m.is_undefined()); + match (name, msg) { + (Some(n), Some(m)) => { + let n = n.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + let m = m.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + if n == "Error" { m } else { format!("{n}: {m}") } + } + _ => v.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_else(|_| display.clone()), + } + } + None => { + let first = display.lines().next().unwrap_or("").trim(); + let first = first.split(" at line ").next().unwrap_or(first).trim(); + let first = match first.rfind(" (") { + Some(i) if first.ends_with(')') && first[i..].contains(':') => first[..i].trim(), + _ => first, + }; + first.strip_prefix("Error: ").unwrap_or(first).to_string() + } + }; + let (line, column) = position(&display, file, code); + ScriptError { message, line, column, file: file.to_string() } +} + +/// Line and column of an error: `at line N, col M` (syntax errors), else the innermost backtrace +/// frame in the user's script (`file:line:col`), else (names boa can't place) the first use of +/// the undefined name in `code`. +fn position(msg: &str, file: &str, code: &str) -> (Option, Option) { + if let Some(i) = msg.find("at line ") { + let rest = &msg[i + 8..]; + let line = rest.split(|c: char| !c.is_ascii_digit()).next().and_then(|n| n.parse().ok()); + let col = rest.find("col ").and_then(|j| rest[j + 4..].split(|c: char| !c.is_ascii_digit()).next()).and_then(|n| n.parse().ok()); + return (line, col); + } + // `(file:12:5)`: the digits after the file name. + let pick = |l: &str| -> Option<(u32, u32)> { + let i = l.find(&format!("({file}:"))? + file.len() + 2; + let mut it = l[i..].split([':', ')']); + Some((it.next()?.parse().ok()?, it.next()?.parse().ok()?)) + }; + let frames: Vec<&str> = msg.lines().skip(1).collect(); + if let Some((a, b)) = frames.iter().find_map(|l| pick(l)).or_else(|| msg.lines().next().and_then(pick)) { + return (Some(a), Some(b)); + } + // `ReferenceError: name is not defined` without a position. + let first = msg.lines().next().unwrap_or(""); + if let Some(name) = first.split("ReferenceError: ").nth(1).and_then(|r| r.split(" is not defined").next()) { + for (n, line) in code.lines().enumerate() { + let b = line.as_bytes(); + let mut from = 0; + while let Some(i) = line[from..].find(name).map(|i| i + from) { + let before = i.checked_sub(1).map(|j| b[j]); + let after = b.get(i + name.len()).copied(); + let ident = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; + if !before.is_some_and(|c| ident(c) || c == b'.') && !after.is_some_and(ident) { + return (Some(n as u32 + 1), Some(i as u32 + 1)); + } + from = i + name.len(); + } + } + } + (None, None) +} + +/// Evaluate `code` in `ctx` and convert the result. +fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { + let src = Source::from_bytes(code.as_bytes()).with_path(std::path::Path::new(file)); + let script = boa_engine::Script::parse(src, None, ctx).map_err(|e| script_error(&e, ctx, file, code))?; + let v = script.evaluate(ctx).map_err(|e| script_error(&e, ctx, file, code))?; + Ok(result_json(&v, ctx)) +} + +/// The script's completion value as JSON (`__result` in the prelude does the conversion). +fn result_json(v: &JsValue, ctx: &mut Context) -> J { + if v.is_undefined() { + return J::Null; + } + let global = ctx.global_object(); + let Some(f) = global.get(js_string!("__result"), ctx).ok().and_then(|f| f.as_object()) else { return J::Null }; + match f.call(&JsValue::undefined(), std::slice::from_ref(v), ctx) { + Ok(s) if s.is_string() => { + let s = s.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + serde_json::from_str(&s).unwrap_or(J::String(s)) + } + _ => J::Null, + } +} + +/// Restores the caller's session when a run ends (on every exit path). +struct Restore<'a> { + target: &'a mut Session, +} + +impl Drop for Restore<'_> { + fn drop(&mut self) { + if let Some(a) = ACTIVE.with(|c| c.borrow_mut().take()) { + *self.target = a.session; + } + } +} + +/// Run a script against `session`: the object model sees and edits that session through engine +/// commands. Script errors come back in [`Outcome::error`]. +pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { + let running = ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)); + if running { + return Outcome { + error: Some(ScriptError { message: "scripts can't run scripts".into(), file: req.name.into(), ..Default::default() }), + ..Default::default() + }; + } + let s = std::mem::take(session); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active { session: s, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: req.name.to_string() })); + let guard = Restore { target: session }; + let file = if req.name.is_empty() { "script" } else { req.name }; + let mut result = if req.console { + CONSOLE.with(|c| { + let mut c = c.borrow_mut(); + if c.is_none() { + match new_context() { + Ok(ctx) => *c = Some(ManuallyDrop::new(ctx)), + Err(e) => return Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + } + } + let ctx = c.as_mut().map(|m| &mut **m).expect("console context"); + eval_in(ctx, req.code, file) + }) + } else { + match new_context() { + Ok(mut ctx) => { + let r = eval_in(&mut ctx, req.code, file); + // app.scheduleTask: run the queued one-shot tasks after the script, in order. + let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); + let mut r = r; + for (_, code, repeat) in tasks { + if repeat { + continue; + } + if let Err(e) = eval_in(&mut ctx, &code, "scheduled task") + && r.is_ok() + { + r = Err(e); + } + } + r + } + Err(e) => Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + } + }; + // Close undo groups left open; drop selections that no longer exist. + let output = with_active(|a| { + if let Some((name, snap)) = a.groups.first().cloned() { + collapse(&mut a.session, &name, &snap); + a.groups.clear(); + } + a.session.sanitize_state(); + Ok(std::mem::take(&mut a.output)) + }) + .unwrap_or_default(); + drop(guard); + let mut out = Outcome { output, ..Default::default() }; + match result.as_mut() { + Ok(v) => out.result = std::mem::take(v), + Err(e) => out.error = Some(e.clone()), + } + out +} diff --git a/crates/script/src/tests.rs b/crates/script/src/tests.rs new file mode 100644 index 000000000..f4a4fcb6f --- /dev/null +++ b/crates/script/src/tests.rs @@ -0,0 +1,392 @@ +//! End-to-end scripts against a headless session: build with the object model, read back through +//! the engine. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::keyframe::Value as KV; +use effectcraft_engine::project::{Comp, ItemId, Layer}; +use serde_json::{Value, json}; + +use crate::{Outcome, run_code}; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), ..Default::default() }; + crate::install(&mut s); + s +} + +fn ok(s: &mut Session, code: &str) -> Outcome { + let o = run_code(s, code, "test.jsx"); + assert!(o.error.is_none(), "script failed: {:?}\noutput: {:?}", o.error, o.output); + o +} + +fn comp_named<'a>(s: &'a Session, name: &str) -> (ItemId, &'a Comp) { + let it = s.project.items.values().find(|i| i.name == name && i.as_comp().is_some()).unwrap_or_else(|| panic!("no comp {name}")); + (it.id, it.as_comp().unwrap()) +} + +fn layer<'a>(c: &'a Comp, name: &str) -> &'a Layer { + c.layers.iter().find(|l| l.name == name).unwrap_or_else(|| panic!("no layer {name}")) +} + +#[test] +fn builds_a_comp_end_to_end() { + let mut s = session(); + let o = ok( + &mut s, + r#" + app.beginUndoGroup("Build"); + var comp = app.project.items.addComp("Main", 640, 360, 1, 4, 25); + var bg = comp.layers.addSolid([0.2, 0.4, 0.6], "BG", 640, 360, 1); + var txt = comp.layers.addText("Hello"); + txt.name = "Title"; + var pos = txt.property("ADBE Transform Group").property("ADBE Position"); + pos.setValueAtTime(0, [100, 180]); + pos.setValueAtTime(2, [540, 180]); + txt.transform.opacity.expression = "50 + 25"; + var blur = bg.Effects.addProperty("ADBE Gaussian Blur 2"); + blur.property("Blurriness").setValue(12); + var nul = comp.layers.addNull(); + txt.setParentWithJump(nul); + bg.blendingMode = BlendingMode.MULTIPLY; + var rq = app.project.renderQueue.items.add(comp); + rq.outputModule(1).file = new File("/tmp/effectcraft-script-test/out.mp4"); + app.endUndoGroup(); + writeLn("layers: " + comp.numLayers); + [comp.numLayers, pos.numKeys, pos.keyValue(2)[0], txt.transform.opacity.value, comp.layer(1).name] + "#, + ); + assert_eq!(o.output, vec!["layers: 3"]); + assert_eq!(o.result, json!([3, 2, 540, 75, "Null 1"])); + let (cid, c) = comp_named(&s, "Main"); + assert_eq!((c.width, c.height), (640, 360)); + assert_eq!(c.duration.seconds(), 4.0); + // AE adds new layers at the top. + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Null 1", "Title", "BG"]); + let title = layer(c, "Title"); + let p = title.props.prop("transform/position").unwrap(); + assert_eq!(p.keys.len(), 2); + assert_eq!(p.keys[1].value, KV::Vec3([540.0, 180.0, 0.0])); + assert_eq!(title.props.prop("transform/opacity").unwrap().expr.as_ref().unwrap().text, "50 + 25"); + assert_eq!(title.parent, Some(layer(c, "Null 1").id)); + let bg = layer(c, "BG"); + assert_eq!(bg.blend_mode, effectcraft_engine::color::BlendMode::Multiply); + let fx = bg.props.group("effects/#1").unwrap(); + assert_eq!(fx.match_id, "ec.blur.gaussian"); + assert_eq!(bg.props.prop("effects/#1/blurriness").unwrap().value, KV::Scalar(12.0)); + assert_eq!(s.project.render_queue.len(), 1); + assert_eq!(s.project.render_queue[0].comp, cid); + // One undo step for the whole group. + assert_eq!(s.history.undo.len(), 1); + assert_eq!(s.history.undo[0].0, "Build"); + assert!(s.undo()); + assert!(s.project.items.is_empty()); +} + +#[test] +fn undo_steps_without_a_group() { + let mut s = session(); + ok(&mut s, r#"var c = app.project.items.addComp("A", 100, 100, 1, 1, 24); c.layers.addNull(); c.layers.addNull();"#); + assert_eq!(s.history.undo.len(), 3); + // An unbalanced group is closed when the script ends. + ok(&mut s, r#"app.beginUndoGroup("Two"); var c = app.project.item(1); c.layers.addNull(); c.layers.addNull();"#); + assert_eq!(s.history.undo.len(), 4); + assert_eq!(s.history.undo[3].0, "Two"); + // Nested groups fold into the outermost one. + ok( + &mut s, + r#"app.beginUndoGroup("Outer"); app.beginUndoGroup("Inner"); app.project.item(1).layers.addNull(); app.endUndoGroup(); app.project.item(1).layers.addNull(); app.endUndoGroup();"#, + ); + assert_eq!(s.history.undo.len(), 5); + assert_eq!(s.history.undo[4].0, "Outer"); + s.undo(); + assert_eq!(s.project.comp(ItemId(1)).unwrap().layers.len(), 4); +} + +#[test] +fn errors_report_line_and_column() { + let mut s = session(); + let o = run_code(&mut s, "var a = 1;\nvar b = 2;\nfoo.bar();\n", "broken.jsx"); + let e = o.error.expect("error"); + assert_eq!(e.line, Some(3), "{e:?}"); + assert!(e.message.contains("foo"), "{e:?}"); + assert_eq!(e.file, "broken.jsx"); + let o = run_code(&mut s, "var a = 1;\n\nvar = ;\n", "syntax.jsx"); + let e = o.error.expect("syntax error"); + assert_eq!(e.line, Some(3), "{e:?}"); + let o = run_code(&mut s, "writeLn('before');\nthrow new Error('custom failure');\n", "throw.jsx"); + let e = o.error.expect("thrown"); + assert_eq!(e.message, "custom failure"); + assert_eq!(e.line, Some(2), "{e:?}"); + assert_eq!(o.output, vec!["before"]); + // Errors from the object model point at the user's line. + let o = run_code(&mut s, "var c = app.project.items.addComp('C', 10, 10, 1, 1, 24);\n\nc.layer(5);\n", "range.jsx"); + let e = o.error.expect("range error"); + assert!(e.message.contains("out of range"), "{e:?}"); + assert_eq!(e.line, Some(3), "{e:?}"); +} + +#[test] +fn match_names_and_lookups() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("M", 200, 200, 1, 2, 30); + var l = c.layers.addSolid([1, 1, 1], "S", 200, 200, 1); + var tr = l.property("ADBE Transform Group"); + var fx = l.property("ADBE Effect Parade").addProperty("Gaussian Blur"); + var m = l.property("ADBE Mask Parade").addProperty("ADBE Mask Atom"); + var sh = c.layers.addShape(); + var grp = sh.property("ADBE Root Vectors Group").addProperty("ADBE Vector Group"); + var rect = grp.property("ADBE Vectors Group").addProperty("ADBE Vector Shape - Rect"); + var fill = grp.property("Contents").addProperty("ADBE Vector Graphic - Fill"); + fill.property("ADBE Vector Fill Color").setValue([1, 0, 0]); + rect.property("ADBE Vector Rect Size").setValue([50, 60]); + [ + l.matchName, tr.matchName, tr.property("ADBE Position").matchName, tr.property("Anchor Point").matchName, + tr.property(1).name, tr.anchorPoint.matchName, l.transform.position.propertyValueType == PropertyValueType.ThreeD_SPATIAL, + fx.matchName, fx.property(1).matchName, fx.property("ADBE Gaussian Blur 2-0001").name, m.matchName, + m.property("ADBE Mask Shape").name, sh.matchName, grp.matchName, rect.matchName, fill.matchName, + tr.propertyType == PropertyType.NAMED_GROUP, l.Effects.propertyType == PropertyType.INDEXED_GROUP, + tr.position.parentProperty.matchName, tr.position.propertyDepth, l.property("Nope") === null, + fx.isEffect, m.isMask, l.Effects.numProperties + ] + "#, + ); + assert_eq!( + o.result, + json!([ + "ADBE AV Layer", + "ADBE Transform Group", + "ADBE Position", + "ADBE Anchor Point", + "Anchor Point", + "ADBE Anchor Point", + true, + "ADBE Gaussian Blur 2", + "ADBE Gaussian Blur 2-0001", + "Blurriness", + "ADBE Mask Atom", + "Mask Path", + "ADBE Vector Layer", + "ADBE Vector Group", + "ADBE Vector Shape - Rect", + "ADBE Vector Graphic - Fill", + true, + true, + "ADBE Transform Group", + 2, + true, + true, + true, + 1 + ]) + ); + let (_, c) = comp_named(&s, "M"); + let sh = c.layers.iter().find(|l| l.props.sub("contents").is_some()).unwrap(); + assert_eq!(sh.props.prop("contents/#1/contents/rect/size").unwrap().value, KV::Vec2([50.0, 60.0])); + assert_eq!(sh.props.prop("contents/#1/contents/fill/color").unwrap().value, KV::Color([1.0, 0.0, 0.0, 1.0])); +} + +#[test] +fn keyframes_eases_and_interpolation() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("K", 100, 100, 1, 5, 25); + var l = c.layers.addNull(); + var r = l.transform.rotation; + r.setValuesAtTimes([0, 1, 2], [0, 90, 180]); + r.setInterpolationTypeAtKey(2, KeyframeInterpolationType.HOLD, KeyframeInterpolationType.LINEAR); + r.setTemporalEaseAtKey(3, [new KeyframeEase(0, 75)]); + var idx = r.addKey(1.5); + r.removeKey(1); + var out = [r.numKeys, idx, r.keyTime(1), r.keyInInterpolationType(1) == KeyframeInterpolationType.HOLD, + r.keyOutInterpolationType(1) == KeyframeInterpolationType.LINEAR, r.keyInTemporalEase(3)[0].influence, + r.valueAtTime(1, false), r.isTimeVarying, r.nearestKeyIndex(1.9)]; + l.startTime = 1; + out.push(r.keyTime(1)); + out + "#, + ); + let v = o.result.as_array().unwrap(); + assert_eq!(v[0], json!(3)); + assert_eq!(v[1], json!(3)); + assert_eq!(v[2], json!(1)); + assert_eq!(v[3], json!(true)); + assert_eq!(v[4], json!(true)); + assert!((v[5].as_f64().unwrap() - 75.0).abs() < 1e-6, "{v:?}"); + assert_eq!(v[6], json!(90)); + assert_eq!(v[7], json!(true)); + assert_eq!(v[8], json!(3)); + // Moving the layer moves its keys (layer time): key 1 now sits at 2 s comp time. + assert_eq!(v[9], json!(2)); +} + +#[test] +fn text_documents() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("T", 400, 200, 1, 2, 30); + var t = c.layers.addText("abc"); + var st = t.property("ADBE Text Properties").property("ADBE Text Document"); + var doc = st.value; + doc.text = "Changed"; + doc.fontSize = 40; + doc.fillColor = [1, 0, 0]; + doc.justification = ParagraphJustification.CENTER_JUSTIFY; + doc.applyStroke = true; + doc.strokeWidth = 3; + st.setValue(doc); + var d2 = t.text.sourceText.value; + [d2.text, d2.fontSize, d2.fillColor, d2.justification == ParagraphJustification.CENTER_JUSTIFY, d2.strokeWidth, t instanceof TextLayer, t.name] + "#, + ); + assert_eq!(o.result, json!(["Changed", 40, [1, 0, 0], true, 3, true, "abc"])); + let (_, c) = comp_named(&s, "T"); + let KV::Text(d) = &c.layers[0].props.prop("text/sourceText").unwrap().value else { panic!() }; + assert_eq!(d.text, "Changed"); + assert_eq!(d.style_at(0).size, 40.0); +} + +#[test] +fn layers_items_and_markers() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var f = app.project.items.addFolder("Comps"); + var c = f.items.addComp("Inner", 100, 100, 1, 3, 30); + var main = app.project.items.addComp("Outer", 200, 200, 1, 6, 30); + var nested = main.layers.add(c); + var a = main.layers.addSolid([1, 0, 0], "A", 50, 50, 1, 2); + var b = main.layers.addSolid([0, 1, 0], "B", 50, 50, 1); + b.moveAfter(nested); + a.enabled = false; a.solo = true; a.locked = true; a.shy = true; + a.threeDLayer = true; + var cam = main.layers.addCamera("Cam", [100, 100]); + var lt = main.layers.addLight("Key", [100, 100]); + lt.lightType = LightType.SPOT; + var d = a.duplicate(); + d.name = "A copy"; + var mv = new MarkerValue("hello"); + mv.duration = 0.5; + main.markerProperty.setValueAtTime(1, mv); + b.marker.setValueAtTime(0.5, new MarkerValue("layer marker")); + b.inPoint = 1; b.outPoint = 3; + var pc = main.layers.precompose([b.index], "Pre", true); + [c.parentFolder.name, nested.source.name, a.outPoint, a.enabled, a.solo, a.locked, a.shy, a.threeDLayer, + cam instanceof CameraLayer, lt.lightType == LightType.SPOT, main.markerProperty.numKeys, main.markerProperty.keyValue(1).comment, + main.markerProperty.keyValue(1).duration, pc.name, pc instanceof CompItem, main.layer("A copy").index, app.project.numItems, + main.layers.byName("Pre") !== null, f.numItems] + "#, + ); + assert_eq!(o.result, json!(["Comps", "Inner", 2, false, true, true, true, true, true, true, 1, "hello", 0.5, "Pre", true, 3, 7, true, 1])); + let (_, c) = comp_named(&s, "Outer"); + assert_eq!(c.markers.len(), 1); + assert!( + c.layers + .iter() + .any(|l| l.name == "Key" + && matches!(l.source, effectcraft_engine::project::LayerSource::Light { kind: effectcraft_engine::project::LightKind::Spot })) + ); +} + +#[test] +fn security_gate_for_files() { + let dir = std::env::temp_dir().join(format!("ec-script-gate-{}", std::process::id())); + let proj = dir.join("proj"); + std::fs::create_dir_all(&proj).unwrap(); + std::fs::write(proj.join("data.txt"), "inside").unwrap(); + std::fs::write(dir.join("secret.txt"), "outside").unwrap(); + let mut s = session(); + s.path = Some(proj.join("p.ecproj").to_string_lossy().to_string()); + let q = |p: &std::path::Path| serde_json::to_string(&p.to_string_lossy()).unwrap(); + // Reading inside the project folder works. + let o = ok(&mut s, &format!("var f = new File({}); f.open('r'); var t = f.read(); f.close(); t", q(&proj.join("data.txt")))); + assert_eq!(o.result, json!("inside")); + // Outside it (also via `..`) is refused. + for path in [dir.join("secret.txt"), proj.join("../secret.txt")] { + let o = run_code(&mut s, &format!("var f = new File({}); f.open('r');", q(&path)), "gate.jsx"); + let e = o.error.expect("blocked"); + assert!(e.message.contains("Allow Scripts to Write Files"), "{e:?}"); + } + // Writing is refused anywhere, the network too. + let o = run_code(&mut s, &format!("var f = new File({}); f.open('w'); f.write('x'); f.close();", q(&proj.join("out.txt"))), "gate.jsx"); + assert!(o.error.is_some()); + assert!(!proj.join("out.txt").exists()); + assert!(run_code(&mut s, "new Socket()", "net.jsx").error.is_some()); + assert!(run_code(&mut s, "system.callSystem('ls')", "sys.jsx").error.is_some()); + // With the preference on, scripts can write and read anywhere. + s.prefs.scripting.allow_scripts_write_files = true; + ok(&mut s, &format!("var f = new File({}); f.open('w'); f.writeln('written'); f.close();", q(&dir.join("w.txt")))); + assert_eq!(std::fs::read_to_string(dir.join("w.txt")).unwrap(), "written\n"); + let o = ok(&mut s, &format!("var f = new File({}); f.open('r'); f.readln()", q(&dir.join("secret.txt")))); + assert_eq!(o.result, json!("outside")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn engine_entry_points() { + let mut s = session(); + // script.run reports errors in the result, not as a command failure. + let r = s.execute("script.run", json!({"code": "writeLn('hi'); 1 + 2"})).unwrap(); + assert_eq!(r["ok"], json!(true)); + assert_eq!(r["result"], json!(3)); + assert_eq!(r["output"], json!("hi")); + let r = s.execute("script.run", json!({"code": "\nnope()"})).unwrap(); + assert_eq!(r["ok"], json!(false)); + assert_eq!(r["error"]["line"], json!(2)); + // Scripts can't run scripts. + let r = s.execute("script.run", json!({"code": "app.run('script.run', {code: '1'})"})).unwrap(); + assert!(r["error"]["message"].as_str().unwrap().contains("scripts can't run scripts"), "{r}"); + // File ▸ Scripts ▸ Run Script File… runs .jsx as JavaScript. + let dir = std::env::temp_dir().join(format!("ec-script-file-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("make.jsx"); + std::fs::write(&f, "app.project.items.addComp('From File', 320, 240, 1, 2, 24);").unwrap(); + s.execute("file.runScript", json!({"path": f.to_string_lossy()})).unwrap(); + assert!(s.project.items.values().any(|i| i.name == "From File")); + std::fs::write(&f, "\n\nbroken(;").unwrap(); + let e = s.execute("file.runScript", json!({"path": f.to_string_lossy()})).unwrap_err().to_string(); + assert!(e.contains("make.jsx (line 3)"), "{e}"); + // JSON command scripts still work. + let j = dir.join("steps.json"); + std::fs::write(&j, r#"[{"command": "comp.new", "params": {"name": "From Steps"}}]"#).unwrap(); + s.execute("file.runScript", json!({"path": j.to_string_lossy()})).unwrap(); + assert!(s.project.items.values().any(|i| i.name == "From Steps")); + // Menu commands by name. + let o = ok(&mut s, "app.findMenuCommandId('New Project') > 0 && app.findMenuCommandId('No Such Thing') === 0"); + assert_eq!(o.result, json!(true)); + let o = ok(&mut s, "app.executeCommand(app.findMenuCommandId('New Project')); app.project.numItems"); + assert_eq!(o.result, json!(0)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn console_keeps_variables() { + let mut s = session(); + let req = |code: &'static str| effectcraft_engine::ScriptRequest { code, name: "console", console: true }; + let r = crate::run(&mut s, &req("var counter = 41;")); + assert!(r.error.is_none(), "{r:?}"); + let r = crate::run(&mut s, &req("counter + 1")); + assert_eq!(r.result, json!(42)); + let _ = Value::Null; +} + +#[test] +fn schedule_task_runs_after_the_script() { + let mut s = session(); + let o = ok( + &mut s, + "app.scheduleTask('writeLn(\"later\")', 100, false); var id = app.scheduleTask('writeLn(\"never\")', 10, false); app.cancelTask(id); writeLn('now');", + ); + assert_eq!(o.output, vec!["now", "later"]); +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 99bc7c5e4..d1a409460 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -34,10 +34,11 @@ pub enum PanelKind { History, Markers, MaskInterpolation, + ScriptConsole, } impl PanelKind { - pub const ALL: [PanelKind; 24] = [ + pub const ALL: [PanelKind; 25] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -62,6 +63,7 @@ impl PanelKind { PanelKind::History, PanelKind::Markers, PanelKind::MaskInterpolation, + PanelKind::ScriptConsole, ]; pub fn title(self) -> &'static str { match self { @@ -89,6 +91,7 @@ impl PanelKind { PanelKind::History => "History", PanelKind::Markers => "Markers", PanelKind::MaskInterpolation => "Mask Interpolation", + PanelKind::ScriptConsole => "Script Console", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2d..32d471b31 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -697,7 +697,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "render.saveCurrentPreview" => ("path", Ask::Save("Preview.mp4")), - "file.runScript" => ("path", Ask::Open(&["jsonl", "json", "txt"])), + "file.runScript" => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), _ => return None, diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..bdd053052 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -21,6 +21,7 @@ pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; +pub mod script_console; pub mod settings; pub mod shortcut_editor; pub mod text_panels; @@ -82,6 +83,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::History => misc::history(app, ui, rect), PanelKind::Markers => misc::markers(app, ui, rect), PanelKind::MaskInterpolation => anim_tools::mask_interpolation(app, ui, rect), + PanelKind::ScriptConsole => script_console::show(app, ui, rect), PanelKind::Wiggler => anim_tools::wiggler(app, ui, rect), PanelKind::Smoother => anim_tools::smoother(app, ui, rect), PanelKind::MotionSketch => anim_tools::motion_sketch(app, ui, rect), diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 1824fec50..e9a40e61c 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -529,16 +529,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } dresp.on_hover_text("Project color depth: click to cycle 8/16/32 bpc, Alt-click for Project Settings"); let tr = Rect::from_min_size(pos2(foot.max.x - 30.0, foot.min.y + 3.0), vec2(22.0, 22.0)); - if widgets::icon_button(ui, tr, Icon::Trash, false, &t, egui::Id::new("ptrash")).on_hover_text("Delete selected project items").clicked() { - let sel = app.session.state.project_selection.clone(); - let _ = app.session.edit("Delete Items", None, |proj, st| { - for id in &sel { - proj.items.remove(id); - } - st.project_selection.clear(); - Ok(()) - }); - app.session.sanitize_state(); + if widgets::icon_button(ui, tr, Icon::Trash, false, &t, egui::Id::new("ptrash")).on_hover_text("Delete selected project items").clicked() + && !app.session.state.project_selection.is_empty() + { + actions.push(("project.delete".into(), json!({}))); } app.auto.add("project.delete", tr, "Delete"); for (id, params) in actions { diff --git a/crates/ui-egui/src/panels/script_console.rs b/crates/ui-egui/src/panels/script_console.rs new file mode 100644 index 000000000..3c258f6ad --- /dev/null +++ b/crates/ui-egui/src/panels/script_console.rs @@ -0,0 +1,144 @@ +//! Window ▸ Script Console: a JavaScript REPL over the After Effects-style scripting object model +//! (`app.project`, comps, layers, properties…). Each run goes through the `script.run` command +//! in the console's persistent context, so variables survive between runs; edits are undoable. + +use egui::{Align2, Rect, Sense, pos2, vec2}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// One line of console output. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ConsoleLine { + pub text: String, + /// `input`, `output`, `result` or `error`. + pub kind: String, +} + +/// Script Console state (serde, so agents can read it). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ScriptConsole { + pub input: String, + pub log: Vec, + /// Previously run inputs (Up/Down recall). + #[serde(default)] + pub history: Vec, +} + +impl ScriptConsole { + fn push(&mut self, kind: &str, text: &str) { + for l in text.lines() { + self.log.push(ConsoleLine { text: l.to_string(), kind: kind.to_string() }); + } + if self.log.len() > 2000 { + let extra = self.log.len() - 2000; + self.log.drain(..extra); + } + } + + /// Record a `script.run` result. + pub fn record(&mut self, code: &str, r: &Result) { + self.push("input", &format!("> {}", code.trim())); + match r { + Ok(v) => { + if let Some(o) = v["output"].as_str().filter(|o| !o.is_empty()) { + self.push("output", o); + } + if let Some(e) = v.get("error").filter(|e| !e.is_null()) { + let line = e["line"].as_u64().map(|l| format!("line {l}: ")).unwrap_or_default(); + self.push("error", &format!("{line}{}", e["message"].as_str().unwrap_or("error"))); + } else if !v["result"].is_null() { + self.push("result", &v["result"].to_string()); + } + } + Err(e) => self.push("error", e), + } + } +} + +/// Run the console input. +pub fn run(app: &mut EffectcraftApp, ctx: &egui::Context) { + let code = app.ui.script_console.input.trim().to_string(); + if code.is_empty() { + return; + } + let r = crate::menus::invoke(app, ctx, "script.run", json!({"code": code, "name": "Script Console", "console": true})); + let con = &mut app.ui.script_console; + con.record(&code, &r); + if con.history.last() != Some(&code) { + con.history.push(code); + } + con.input.clear(); +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let input_h = 86.0; + let bar_h = 26.0; + let out_rect = Rect::from_min_max(rect.min, pos2(rect.max.x, rect.max.y - input_h - bar_h)); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(out_rect, 0.0, t.field_bg); + // Output, newest at the bottom. + let line_h = 16.0; + let visible = ((out_rect.height() - 8.0) / line_h).max(1.0) as usize; + let log = &app.ui.script_console.log; + let start = log.len().saturating_sub(visible); + for (i, l) in log[start..].iter().enumerate() { + let color = match l.kind.as_str() { + "error" => t.danger, + "input" => t.text_faint, + "result" => t.accent, + _ => t.text, + }; + p.text(pos2(out_rect.min.x + 8.0, out_rect.min.y + 4.0 + i as f32 * line_h), Align2::LEFT_TOP, &l.text, Tokens::mono(11.5), color); + } + if log.is_empty() { + p.text( + out_rect.center(), + Align2::CENTER_CENTER, + "JavaScript with the After Effects scripting object model — try app.project.numItems", + Tokens::ui(12.0), + t.text_faint, + ); + } + app.auto.add("scriptConsole.output", out_rect, "Script Console output"); + // Input. + let in_rect = Rect::from_min_size(pos2(rect.min.x + 4.0, out_rect.max.y + 4.0), vec2(rect.width() - 8.0, input_h - 8.0)); + let mut run_now = false; + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(in_rect)); + let te = egui::TextEdit::multiline(&mut app.ui.script_console.input) + .id(egui::Id::new("script-console-input")) + .font(Tokens::mono(12.0)) + .desired_width(in_rect.width()) + .desired_rows(4) + .hint_text("Script… (⌘/Ctrl+Enter runs)"); + let resp = child.add_sized(in_rect.size(), te); + if resp.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && i.modifiers.command) { + run_now = true; + } + app.auto.add("scriptConsole.input", in_rect, "Script Console input"); + // Buttons. + let bar = Rect::from_min_size(pos2(rect.min.x, rect.max.y - bar_h), vec2(rect.width(), bar_h)); + let run_r = Rect::from_min_size(pos2(bar.max.x - 64.0, bar.min.y + 2.0), vec2(56.0, 20.0)); + let clear_r = Rect::from_min_size(pos2(bar.max.x - 128.0, bar.min.y + 2.0), vec2(56.0, 20.0)); + let run_resp = ui.interact(run_r, egui::Id::new("script-console-run"), Sense::click()); + p.rect_filled(run_r, 3.0, if run_resp.hovered() { t.accent_hover } else { t.accent }); + p.text(run_r.center(), Align2::CENTER_CENTER, "Run", Tokens::ui(12.0), egui::Color32::WHITE); + let clear_resp = ui.interact(clear_r, egui::Id::new("script-console-clear"), Sense::click()); + p.rect_stroke(clear_r, 3.0, egui::Stroke::new(1.0, t.field_border), egui::StrokeKind::Inside); + p.text(clear_r.center(), Align2::CENTER_CENTER, "Clear", Tokens::ui(12.0), t.text); + app.auto.add("scriptConsole.run", run_r, "Run"); + app.auto.add("scriptConsole.clear", clear_r, "Clear"); + if run_resp.clicked() { + run_now = true; + } + if clear_resp.clicked() { + app.ui.script_console.log.clear(); + } + if run_now { + let ctx = ui.ctx().clone(); + run(app, &ctx); + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d13501032..7d423b8a9 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -476,6 +476,9 @@ impl Default for AnimToolsState { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct UiState { pub tool: Tool, + /// Window ▸ Script Console input, output and history. + #[serde(default)] + pub script_console: crate::panels::script_console::ScriptConsole, /// Tool shown in each toolbar slot. pub slot_tools: Vec, pub workspace: String, @@ -576,6 +579,7 @@ impl Default for UiState { fn default() -> Self { UiState { tool: Tool::Selection, + script_console: Default::default(), slot_tools: Tool::SLOTS.iter().map(|s| s[0]).collect(), workspace: "Default".into(), dock: crate::dock::workspace("Default"), diff --git a/crates/ui-egui/tests/ui_script_console.rs b/crates/ui-egui/tests/ui_script_console.rs new file mode 100644 index 000000000..a8068c9b3 --- /dev/null +++ b/crates/ui-egui/tests/ui_script_console.rs @@ -0,0 +1,44 @@ +//! Window ▸ Script Console: the panel shows, runs JavaScript through `script.run` (persistent +//! context) and logs output, results and errors (egui_kittest, UI logic only). + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")).clone(); + let c = egui::pos2(e.rect[0] + e.rect[2] / 2.0, e.rect[1] + e.rect[3] / 2.0); + h.input_mut().events.push(egui::Event::PointerMoved(c)); + for pressed in [true, false] { + h.input_mut().events.push(egui::Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(2); +} + +#[test] +fn script_console_runs_scripts() { + let s = effectcraft_host::session(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::ScriptConsole); + h.run_steps(3); + for id in ["scriptConsole.input", "scriptConsole.output", "scriptConsole.run", "scriptConsole.clear"] { + assert!(h.state().auto.find(id).is_some(), "missing {id}"); + } + h.state_mut().ui.script_console.input = "var c = app.project.items.addComp('Console', 320, 180, 1, 2, 24); writeLn('made'); c.name".into(); + h.run_steps(1); + click(&mut h, "scriptConsole.run"); + assert!(h.state().session.project.items.values().any(|i| i.name == "Console")); + // Variables persist between runs; errors are logged with their line. + h.state_mut().ui.script_console.input = "c.numLayers + 5".into(); + click(&mut h, "scriptConsole.run"); + h.state_mut().ui.script_console.input = "nope()".into(); + click(&mut h, "scriptConsole.run"); + let log: Vec<(String, String)> = h.state().ui.script_console.log.iter().map(|l| (l.kind.clone(), l.text.clone())).collect(); + assert!(log.contains(&("output".into(), "made".into())), "{log:?}"); + assert!(log.contains(&("result".into(), "\"Console\"".into())), "{log:?}"); + assert!(log.contains(&("result".into(), "5".into())), "{log:?}"); + assert!(log.iter().any(|(k, t)| k == "error" && t.contains("line 1") && t.contains("nope")), "{log:?}"); + assert!(h.state_mut().session.undo(), "console edits are undoable"); + click(&mut h, "scriptConsole.clear"); + assert!(h.state().ui.script_console.log.is_empty()); +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356d..40dac3c9f 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -49,6 +49,7 @@ The server speaks JSON-RPC 2.0 over stdio, one message per line, and supports MC |---|---| | `list_commands {filter?, enabled_only?}` | Discover command ids, their param docs, and whether each can run now. | | `execute_command {command, params?}` | Run any command (undoable). | +| `run_script {code, name?}` | Run JavaScript with the After Effects-style scripting object model (`app.project`, `comp.layers.addText(…)`, `layer.property("ADBE Transform Group").property("ADBE Position").setValueAtTime(…)`…). Returns `{ok, result, output, error: {message, line, column}}`; edits are undoable. | | `get_project` / `get_comp {comp?}` | Project items, comp settings and layers. | | `get_layer {layer, comp?, time?, flat?}` | A layer's property tree. Every node has a `path`. | | `get_property {layer, path, comp?, time?}` | Value at a time, keyframes and expression. | @@ -191,6 +192,19 @@ effectcraft-cli exec file.exportLottie '{"comp":"Main","path":"main.json","inclu effectcraft-cli exec file.importLottie '{"path":"anim.json"}' main.ecproj --save ``` +Scripts written against After Effects' documented scripting API run with `script`: + +```sh +effectcraft-cli script build.jsx --save-as main.ecproj # empty project unless one is given +effectcraft-cli script --eval 'app.project.item(1).numLayers' main.ecproj --json +effectcraft-cli script tweak.jsx --bridge 9877 # against the live app +``` + +`writeLn`/`$.writeln`/`alert` output is printed, then the value of the last expression. A script +error exits with status 1 and `file:line:col: message`. Scripts may read files only in the +project's folder and may not write files or use the network unless the user turns on Preferences ▸ +Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network. + `file.exportLottie` returns `{path, bytes, warnings}`: the warnings list every feature Lottie cannot express (most effects, cameras and lights, layer styles, audio, video footage…), so an agent can check what a player will not show. A `.lottie` path writes a dotLottie archive. diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb5..dceb0e0cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,13 +30,15 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | -| L4 | `host` | A fully wired `Session` (media, expressions, exporter) for the frontends | +| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) whose edits run engine commands; AE match names | +| L4 | `host` | A fully wired `Session` (media, expressions, scripting, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path, track`, -`media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine`. +`media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine, script`, +`script → engine`. ## 2. Time @@ -182,6 +184,28 @@ Expressions are JavaScript, run by boa, with After Effects' object model (`thisC `time`, `value`, `wiggle`, `loopOut`, vector maths on arrays, and so on). A syntax error keeps the text, disables the expression and shows a warning, like After Effects. +## 5a. Scripting + +`effectcraft-script` runs JavaScript (boa, a fresh context per run; the Script Console keeps one) +with an object model written from the behaviour the After Effects Scripting Guide documents: +`app`, `Project`, `ItemCollection`, `CompItem`/`FootageItem`/`FolderItem`, `LayerCollection`, +`AVLayer`/`TextLayer`/`ShapeLayer`/`CameraLayer`/`LightLayer`, `PropertyGroup`/`Property` +(values, keyframes, eases, expressions), `TextDocument`, `Shape`, `KeyframeEase`, `MarkerValue`, +`RenderQueue`/`RenderQueueItem`/`OutputModule`, `File`/`Folder` and the enums. The model lives in +`prelude.js`; it reads the session through `__query` (JSON snapshots of items, layers, property +nodes, values and keys) and edits only through `__exec`, which runs engine commands, so every +scripted edit is undoable and identical to the UI's. While a script runs the session is moved +into a thread-local the natives borrow (nothing borrowed enters the JS heap); the comp a call +targets is made active for that command only. `app.beginUndoGroup`/`endUndoGroup` fold the undo +steps in between into one named step. Match names (`ADBE Transform Group`, `ADBE Position`, +`ADBE Gaussian Blur 2`, `ADBE Vector Shape - Rect`…) come from a table in `matchnames.rs`; effect +parameters are `-0001…`. `File` reads are limited to the project's folder, and +writes and the network are refused, unless Preferences ▸ Scripting & Expressions ▸ Allow Scripts +to Write Files and Access Network is on. Entry points: the `script.run` command +(`Session::script`, set by the host), File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`; `.json` +command scripts still run as steps), Window ▸ Script Console, `effectcraft-cli script` and the +MCP `run_script` tool. + ## 6. Commands, the UI seam and automation Every action is a command: an id, a label, a menu path, a shortcut, a parameter description, diff --git a/docs/parity.md b/docs/parity.md index d259fec24..02fa21756 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -75,7 +75,7 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, +the new tracking library, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -87,7 +87,7 @@ wasm threads. | Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | | Audio | 85% | 0.5 | audio to keyframes | | Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | -| Automation | 76% | 3.3 | a JavaScript scripting object model | +| Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3d..2532f2a29 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -39,6 +39,7 @@ const LAYERS: &[(&str, u8)] = &[ ("lottie", 3), ("format", 3), ("engine", 4), + ("script", 4), ("host", 4), ("ui-egui", 5), ("automation", 5), @@ -63,6 +64,8 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("gpu", "render"), ("lottie", "format"), ("host", "engine"), + ("script", "engine"), + ("host", "script"), ("cli", "effectcraft"), ]; From d880746fecf0c211c75983de380708e32a54d590 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:18:11 -0400 Subject: [PATCH 05/14] M14.4: scripting tests for layer/comp settings, import + save, console snapshot; Project.saveAs Co-Authored-By: Claude Opus 5.5 --- crates/host/src/lib.rs | 29 ++++++++++++ crates/script/src/prelude.js | 1 + crates/script/src/tests.rs | 56 +++++++++++++++++++++++ crates/ui-egui/tests/ui_script_console.rs | 24 ++++++++++ 4 files changed, 110 insertions(+) diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 5235fecaa..45e850ff1 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -220,6 +220,35 @@ mod tests { assert!(bytes.starts_with(b"GIF89a")); } + /// importFile, layers.add(footage) and project save/open from a script. + #[test] + fn script_imports_footage_and_saves() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-script-import"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let q = |n: &str| serde_json::to_string(&dir.join(n).to_string_lossy()).unwrap(); + let mut s = super::session(); + let code = format!( + r#" + var c = app.project.items.addComp("Src", 32, 32, 1, 1, 10); + c.layers.addSolid([0, 1, 0], "G", 32, 32, 1); + app.run("comp.saveFrameAs", {{comp: c.id, path: {png}}}); + var still = app.project.importFile(new ImportOptions(new File({png}))); + var main = app.project.items.addComp("Main", 64, 64, 1, 1, 10); + var l = main.layers.add(still); + app.project.save(new File({proj})); + [still instanceof FootageItem, still.width, still.mainSource.isStill, l.source.name, app.project.file.name] + "#, + png = q("frame.png"), + proj = q("p.ecproj") + ); + let r = s.execute("script.run", json!({"code": code})).unwrap(); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"], json!([true, 32, true, "frame.png", "p.ecproj"]), "{r}"); + let r = s.execute("script.run", json!({"code": format!("app.open(new File({})); app.project.numItems", q("p.ecproj"))})).unwrap(); + assert_eq!(r["result"], json!(5), "{r}"); // Src, Solids, G, frame.png, Main + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/script/src/prelude.js b/crates/script/src/prelude.js index 4f3599264..c4f256f4b 100644 --- a/crates/script/src/prelude.js +++ b/crates/script/src/prelude.js @@ -1571,6 +1571,7 @@ Project.prototype = { else __call("file.save", {}); return true; }, + saveAs: function (file) { return this.save(file); }, saveWithDialog: function () { if (this.file) return this.save(); return false; }, close: function (opt) { if (opt === CloseOptions.SAVE_CHANGES && this.file) this.save(); diff --git a/crates/script/src/tests.rs b/crates/script/src/tests.rs index f4a4fcb6f..458f2b41d 100644 --- a/crates/script/src/tests.rs +++ b/crates/script/src/tests.rs @@ -390,3 +390,59 @@ fn schedule_task_runs_after_the_script() { ); assert_eq!(o.output, vec!["now", "later"]); } + +#[test] +fn layer_switches_timing_and_comp_settings() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("S", 320, 240, 1, 5, 30); + var matte = c.layers.addText("M"); + var fill = c.layers.addSolid([1, 1, 1], "Fill", 320, 240, 1); + fill.moveToEnd(); + fill.setTrackMatte(matte, TrackMatteType.LUMA); + var t1 = fill.trackMatteType == TrackMatteType.LUMA && fill.hasTrackMatte && fill.trackMatteLayer.name == "M"; + fill.stretch = 200; + fill.label = 3; + c.width = 400; + c.duration = 8; + c.bgColor = [0.1, 0.2, 0.3]; + c.time = 2; + c.workAreaStart = 1; + c.workAreaDuration = 3; + var extra = c.layers.addNull(); + extra.remove(); + fill.selected = true; + var sel = c.selectedLayers.length; + fill.transform.opacity.expression = "thisLayer.nope("; + var err = fill.transform.opacity.expressionError; + fill.transform.opacity.expressionEnabled = false; + var copy = c.duplicate(); + var f = app.project.items.addFolder("Bin"); + copy.parentFolder = f; + [t1, fill.stretch, fill.label, c.width, c.duration, c.bgColor[2], c.time, c.workAreaStart, c.workAreaDuration, + c.numLayers, sel, err.length > 0, fill.transform.opacity.expressionEnabled, copy.name, f.numItems, copy.parentFolder.name] + "#, + ); + let v = o.result; + assert_eq!(v[0], json!(true), "{v}"); + assert_eq!(v[1], json!(200)); + assert_eq!(v[2], json!(3)); + assert_eq!(v[3], json!(400)); + assert_eq!(v[4], json!(8)); + assert!((v[5].as_f64().unwrap() - 0.3).abs() < 1e-6); + assert_eq!(v[6], json!(2)); + assert_eq!(v[7], json!(1)); + assert_eq!(v[8], json!(3)); + assert_eq!(v[9], json!(2)); + assert_eq!(v[10], json!(1)); + assert_eq!(v[11], json!(true)); + assert_eq!(v[12], json!(false)); + assert_eq!(v[13], json!("S 2")); + assert_eq!(v[14], json!(1)); + assert_eq!(v[15], json!("Bin")); + // Removing the copy through the object model. + let o = ok(&mut s, "var n = app.project.numItems; app.project.item(1).remove(); n - app.project.numItems"); + assert_eq!(o.result, json!(1)); +} diff --git a/crates/ui-egui/tests/ui_script_console.rs b/crates/ui-egui/tests/ui_script_console.rs index a8068c9b3..526a52938 100644 --- a/crates/ui-egui/tests/ui_script_console.rs +++ b/crates/ui-egui/tests/ui_script_console.rs @@ -42,3 +42,27 @@ fn script_console_runs_scripts() { click(&mut h, "scriptConsole.clear"); assert!(h.state().ui.script_console.log.is_empty()); } + +/// Headless look at the panel (wgpu offscreen); run with +/// `SC_SNAPSHOT=/abs/out.png cargo test -p effectcraft-ui-egui --test ui_script_console -- --ignored`. +#[test] +#[ignore] +fn script_console_snapshot() { + let s = effectcraft_host::session(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::ScriptConsole); + h.run_steps(2); + for code in ["var c = app.project.items.addComp('Console', 320, 180, 1, 2, 24); writeLn('made ' + c.name); c.id", "c.layers.addText('Hi').name", "nope()"] { + h.state_mut().ui.script_console.input = code.into(); + h.run_steps(1); + click(&mut h, "scriptConsole.run"); + } + h.state_mut().ui.script_console.input = "app.project.numItems".into(); + h.run_steps(3); + let img = h.render().expect("render"); + let out = std::env::var("SC_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out/script_console.png").into()); + if let Some(d) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(d).unwrap(); + } + img.save(&out).unwrap(); +} From 07330bca6b15da7653a95c7255a0e923c85837b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:20:25 -0400 Subject: [PATCH 06/14] M6.7: roto brush segmentation + propagation Graph-cut segmentation (Boykov-Kolmogorov max-flow, GrabCut GMMs, coarse to fine), optical-flow propagation with band re-segmentation, correction strokes, Roto Brush & Refine Edge effect with content-keyed derived cache, roto.* commands (stroke/propagate/span/freeze/unfreeze/clearStrokes/ cancel/options/status), refine-edge matting (guided filter + closed-form) and decontamination, Freeze job. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/effects/Cargo.toml | 3 +- crates/effects/src/lib.rs | 6 +- crates/effects/src/roto.rs | 698 ++++++++++++++++++++++ crates/engine/src/commands/edit.rs | 3 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/roto_cmds.rs | 424 +++++++++++++ crates/engine/src/lib.rs | 22 + crates/engine/src/roto.rs | 569 ++++++++++++++++++ crates/engine/src/tests_roto.rs | 288 +++++++++ crates/track/Cargo.toml | 2 +- crates/track/src/lib.rs | 1 + crates/track/src/roto/gmm.rs | 228 +++++++ crates/track/src/roto/matting.rs | 465 +++++++++++++++ crates/track/src/roto/maxflow.rs | 438 ++++++++++++++ crates/track/src/roto/mod.rs | 754 ++++++++++++++++++++++++ crates/track/src/roto/refine.rs | 198 +++++++ crates/track/src/roto/rle.rs | 136 +++++ crates/track/src/roto/tests.rs | 331 +++++++++++ 19 files changed, 4565 insertions(+), 4 deletions(-) create mode 100644 crates/effects/src/roto.rs create mode 100644 crates/engine/src/commands/roto_cmds.rs create mode 100644 crates/engine/src/roto.rs create mode 100644 crates/engine/src/tests_roto.rs create mode 100644 crates/track/src/roto/gmm.rs create mode 100644 crates/track/src/roto/matting.rs create mode 100644 crates/track/src/roto/maxflow.rs create mode 100644 crates/track/src/roto/mod.rs create mode 100644 crates/track/src/roto/refine.rs create mode 100644 crates/track/src/roto/rle.rs create mode 100644 crates/track/src/roto/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..544b66157 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1184,6 +1184,7 @@ dependencies = [ "effectcraft-raster", "effectcraft-track", "rayon", + "serde", "serde_json", ] diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index e321fa2f3..20068a93a 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -17,6 +17,5 @@ effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-track = { workspace = true } rayon = { workspace = true } - -[dev-dependencies] +serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a1..2c3255d7d 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -31,6 +31,7 @@ pub mod paint; mod perspective; mod perspective2; pub mod puppet; +pub mod roto; mod sim; mod sim2; mod stylize2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -426,7 +428,7 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS.iter().chain(roto::GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Each frame has its own segmentation. + roto::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs new file mode 100644 index 000000000..a1f93a923 --- /dev/null +++ b/crates/effects/src/roto.rs @@ -0,0 +1,698 @@ +//! Roto Brush & Refine Edge (applied by the Roto Brush tool; Effect ▸ Matte). +//! +//! The instance stores the user's strokes, base frame and segmentation span as JSON in its hidden +//! **Strokes** parameter ([`RotoData`]); the segmentation itself is *derived* data: each frame's +//! result is a deterministic function of its chain key ([`RotoData::chain_keys`] seeded with +//! [`seed`]: what the frames are, the Roto Brush settings and the strokes from the base frame +//! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the +//! engine's propagation job fills at full resolution and rendering fills on demand (reading +//! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project +//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen** +//! parameter ([`FrozenCache`]); they are used while their key still matches. +//! +//! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather, +//! Contrast) → Refine Edge matting inside the Refine Edge band (guided filter + closed-form +//! matting with Higher Quality) → Refine Edge Matte adjustments → matte motion blur → +//! decontamination of the edge colours. Outside the segmentation span the layer is transparent; +//! without strokes the effect passes its input through. +//! +//! *Version* is stored but all versions use the same (classic, graph-cut) engine; *Quality ▸ +//! Best* segments at a higher working resolution with more iterations. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_track::roto::refine::{self, MatteParams}; +use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle}; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +pub use effectcraft_track::roto::{Stroke, StrokeKind}; + +/// The effect's id. +pub const ID: &str = "ec.matte.rotobrush"; +/// Hidden parameters. +pub const STROKES: &str = "strokes"; +pub const INPUT_KEY: &str = "inputKey"; +pub const FROZEN: &str = "frozen"; + +/// Default segmentation span each side of the base frame (frames). +pub const DEFAULT_SPAN: i64 = 20; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("rotoBrushMatte", "Roto Brush Matte"), ("refineEdgeMatte", "Refine Edge Matte")]; + +pub const VERSIONS: [&str; 3] = ["1.0", "2.0", "3.0"]; +pub const QUALITIES: [&str; 2] = ["Standard", "Best"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Roto Brush & Refine Edge", + category: "Matte", + params: vec![ + p("version", "Version", Value::Enum(2), popup(&VERSIONS)), + p("quality", "Quality", Value::Enum(0), popup(&QUALITIES)), + p("rotoBrushMatte/searchRadius", "Search Radius", num(15.0), slider(0.0, 100.0, 0.0, 50.0, 1)), + p("rotoBrushMatte/motionThreshold", "Motion Threshold", num(10.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/motionDamping", "Motion Damping", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/viewSearchRegion", "View Search Region", Value::Bool(false), ParamUi::Checkbox), + p("rotoBrushMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)), + p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("refineEdgeMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/useMotionBlur", "Use Motion Blur", Value::Bool(false), ParamUi::Checkbox), + p("refineEdgeMatte/motionBlurSamples", "Motion Blur Samples", num(11.0), slider(1.0, 64.0, 1.0, 32.0, 0)), + p("refineEdgeMatte/shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 0)), + p("refineEdgeMatte/higherQuality", "Higher Quality", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminateEdgeColors", "Decontaminate Edge Colors", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminationAmount", "Decontamination Amount", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/extendWhereSmoothed", "Extend Where Smoothed", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/increaseDecontaminationRadius", "Increase Decontamination Radius", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/viewDecontaminationMap", "View Decontamination Map", Value::Bool(false), ParamUi::Checkbox), + p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// The instance's strokes (parsed once per content). +pub fn data(params: &Params) -> Arc { + static C: OnceLock>>> = OnceLock::new(); + let j = s(params, STROKES); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return d.clone(); + } + let d = Arc::new(RotoData::from_json(j)); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 64 { + m.clear(); + } + m.insert(h, d.clone()); + d +} + +/// Segmentation settings at `scale` (pixels per layer pixel). +pub fn seg_opts(params: &Params, scale: f64) -> SegOpts { + SegOpts { + scale, + search_radius: f(params, "rotoBrushMatte/searchRadius").max(0.0), + motion_threshold: f(params, "rotoBrushMatte/motionThreshold"), + motion_damping: f(params, "rotoBrushMatte/motionDamping"), + best: e(params, "quality") == 1, + } +} + +/// Matte settings. +pub fn matte_params(params: &Params) -> MatteParams { + MatteParams { + rb_feather: f(params, "rotoBrushMatte/feather").max(0.0), + rb_contrast: f(params, "rotoBrushMatte/contrast"), + rb_shift: f(params, "rotoBrushMatte/shiftEdge"), + rb_chatter: f(params, "rotoBrushMatte/reduceChatter"), + smooth: f(params, "refineEdgeMatte/smooth").max(0.0), + feather: f(params, "refineEdgeMatte/feather").max(0.0), + contrast: f(params, "refineEdgeMatte/contrast"), + shift: f(params, "refineEdgeMatte/shiftEdge"), + chatter: f(params, "refineEdgeMatte/reduceChatter"), + motion_blur: b(params, "refineEdgeMatte/useMotionBlur"), + mb_samples: f(params, "refineEdgeMatte/motionBlurSamples").round().clamp(1.0, 64.0) as usize, + shutter_angle: f(params, "refineEdgeMatte/shutterAngle"), + higher_quality: b(params, "refineEdgeMatte/higherQuality"), + decontaminate: b(params, "refineEdgeMatte/decontaminateEdgeColors"), + decontamination: f(params, "refineEdgeMatte/decontaminationAmount"), + extend_where_smoothed: b(params, "refineEdgeMatte/extendWhereSmoothed"), + increase_radius: f(params, "refineEdgeMatte/increaseDecontaminationRadius").max(0.0), + } +} + +/// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer +/// size, the frame rate and the segmentation settings. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [ + layer_size[0], + layer_size[1], + fps, + f(params, "rotoBrushMatte/searchRadius"), + f(params, "rotoBrushMatte/motionThreshold"), + f(params, "rotoBrushMatte/motionDamping"), + ] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]); + h +} + +/// The layer frame shown at layer time `t` seconds. +pub fn frame_of(t: f64, fps: f64) -> i64 { + (t * fps + 1e-6).floor() as i64 +} + +// ------------------------------------------------------------------ derived-data cache + +struct SegCache { + map: HashMap<(u64, u64), Arc>, + order: VecDeque<(u64, u64)>, + bytes: usize, +} + +const CACHE_BYTES: usize = 768 << 20; + +fn cache() -> &'static Mutex { + static C: OnceLock> = OnceLock::new(); + C.get_or_init(|| Mutex::new(SegCache { map: HashMap::new(), order: VecDeque::new(), bytes: 0 })) +} + +/// A cached segmentation (chain key, scale). +pub fn cached(key: u64, scale: f64) -> Option> { + cache().lock().unwrap_or_else(|e| e.into_inner()).map.get(&(key, scale.to_bits())).cloned() +} + +pub fn store(key: u64, scale: f64, seg: Arc) { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let k = (key, scale.to_bits()); + if c.map.contains_key(&k) { + return; + } + c.bytes += seg.matte.len() + seg.refine.len(); + c.map.insert(k, seg); + c.order.push_back(k); + while c.bytes > CACHE_BYTES { + let Some(old) = c.order.pop_front() else { break }; + if let Some(s) = c.map.remove(&old) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } +} + +/// Drop the cached segmentations of these chain keys (every scale). +pub fn forget(keys: impl IntoIterator) { + let keys: std::collections::HashSet = keys.into_iter().collect(); + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let gone: Vec<(u64, u64)> = c.map.keys().filter(|k| keys.contains(&k.0)).copied().collect(); + for k in gone { + if let Some(s) = c.map.remove(&k) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } + c.order.retain(|k| !keys.contains(&k.0)); +} + +/// Drop every cached segmentation (Edit ▸ Purge). +pub fn purge() { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + c.map.clear(); + c.order.clear(); + c.bytes = 0; +} + +/// The layer-space grid (`round(size · scale)` pixels) of a buffer. +pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image { + let s = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1)); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = buf.img.get(x as i64 + ox, y as i64 + oy); + } + }); + img +} + +/// Everything needed to compute an instance's segmentations. +pub struct Chain { + pub data: Arc, + pub keys: BTreeMap, + pub opts: SegOpts, + pub refine_radius: f64, +} + +impl Chain { + pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain { + let data = data(params); + let keys = data.chain_keys(seed(params, layer_size, fps)); + Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale) } + } + + /// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest + /// cached frame towards the base (full-resolution results serve smaller scales). + /// `frame(g)`: the effect's input on frame g on the layer grid at this scale. + pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool) -> Option> { + let sc = self.opts.scale; + let lookup = |g: i64| -> Option> { + let k = *self.keys.get(&g)?; + if let Some(s) = cached(k, sc) { + return Some(s); + } + if sc != 1.0 { + let full = cached(k, 1.0)?; + let s = Arc::new(downscale(&full, sc)); + store(k, sc, s.clone()); + return Some(s); + } + None + }; + if let Some(s) = lookup(f) { + return Some(s); + } + let base = self.data.base?; + if !self.data.in_span(f) { + return None; + } + // Walk towards the base to the nearest available frame. + let mut path = vec![f]; + let mut g = f; + let mut start: Option> = None; + while let Some(src) = self.data.source_of(g) { + if let Some(s) = lookup(src) { + start = Some(s); + break; + } + path.push(src); + g = src; + } + path.reverse(); + let mut prev_img: Option> = None; + let mut cur = start; + for g in path { + if cancel() { + return None; + } + let img = frame(g)?; + let strokes = self.data.strokes_at(g); + let seg = match (&cur, g == base) { + (_, true) | (None, _) => rb::segment(&img, &strokes, None, &self.opts, self.refine_radius), + (Some(p), false) => { + let src = self.data.source_of(g)?; + let pimg = match prev_img.take() { + Some(i) => i, + None => frame(src)?, + }; + if pimg.width != img.width || pimg.height != img.height { + return None; + } + rb::propagate(&pimg, p, &img, &strokes, &self.opts, self.refine_radius) + } + }; + let seg = Arc::new(seg); + store(*self.keys.get(&g)?, sc, seg.clone()); + cur = Some(seg); + prev_img = Some(img); + } + cur + } + + /// Whether frame `f` is computed (at full resolution or at `scale`). + pub fn is_cached(&self, f: i64, scale: f64) -> bool { + self.keys.get(&f).is_some_and(|k| cached(*k, 1.0).is_some() || cached(*k, scale).is_some()) + } +} + +/// Nearest-neighbour resampling of a segmentation to another scale. +pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { + let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize); + resample(s, w, h) +} + +fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { + if s.w == w && s.h == h { + return s.clone(); + } + let mut out = FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], fg: s.fg.clone(), bg: s.bg.clone() }; + if s.w == 0 || s.h == 0 { + return out; + } + for y in 0..h { + let sy = (((y as f64 + 0.5) * s.h as f64 / h as f64) as usize).min(s.h - 1); + for x in 0..w { + let sx = (((x as f64 + 0.5) * s.w as f64 / w as f64) as usize).min(s.w - 1); + out.matte[y * w + x] = s.matte[sy * s.w + sx]; + out.refine[y * w + x] = s.refine.get(sy * s.w + sx).copied().unwrap_or(0); + } + } + out +} + +/// The final matte of frame `seg` (with its neighbours), on its own grid. +pub fn final_alpha( + img: &Image, + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + mp: &MatteParams, + refine_radius: f64, + scale: f64, +) -> Vec { + let rgb = rb::rgb_of(img); + refine::matte_alpha(&rgb, seg, prev, next, mp, refine_radius * scale, scale) +} + +fn needs_neighbours(mp: &MatteParams) -> bool { + mp.rb_chatter > 0.0 || mp.chatter > 0.0 || mp.motion_blur +} + +// ------------------------------------------------------------------ Freeze + +/// Frozen mattes (Freeze): the final 8-bit alpha and the Refine Edge band of every frame of +/// the span at full resolution, valid while `key` matches [`frozen_key`]. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FrozenCache { + pub key: String, + pub size: [usize; 2], + pub frames: BTreeMap, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FrozenFrame { + /// Alpha (RLE, base64). + pub a: String, + /// Refine Edge band (RLE, base64). + pub b: String, +} + +impl FrozenCache { + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } +} + +/// What frozen mattes depend on: every chain key and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String { + let ch = Chain::new(params, layer_size, fps, 1.0); + let mut h = 0xcbf2_9ce4_8422_2325; + for (f, k) in &ch.keys { + fnv(&mut h, &f.to_le_bytes()); + fnv(&mut h, &k.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +type FrozenParsed = Mutex>>; + +fn frozen(params: &Params) -> Option> { + static C: OnceLock = OnceLock::new(); + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + +/// Encode one frame for [`FrozenCache`]. +pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame { + let a: Vec = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect(); + FrozenFrame { a: rle::encode(&a), b: rle::encode(band) } +} + +// ------------------------------------------------------------------ render + +/// A frame's matte (alpha and band) on the layer grid of `img` at `scale`. +fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc, scale: f64) -> Option<(Vec, Vec, f64)> { + let (w, h) = (img.width as usize, img.height as usize); + // Frozen mattes. + if let Some(fc) = frozen(ctx.params) + && fc.key == frozen_key(ctx.params, ctx.layer_size, ctx.fps()) + && let Some(fr) = fc.frames.get(&f) + { + let a = rle::decode(&fr.a)?; + let bnd = rle::decode(&fr.b)?; + let [fw, fh] = fc.size; + if a.len() == fw * fh && bnd.len() == fw * fh { + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + let (alpha, band) = if fw == w && fh == h { + (af, bnd) + } else { + let mut al = vec![0.0f32; w * h]; + let mut bd = vec![0u8; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + al[y * w + x] = rb::matting::sample(&af, fw, fh, sx, sy); + bd[y * w + x] = bnd[(sy as usize).min(fh - 1) * fw + (sx as usize).min(fw - 1)]; + } + } + (al, bd) + }; + let rr = data(ctx.params).refine_radius(); + return Some((alpha, band, rr)); + } + } + let host = ctx.env.host; + let chain = Chain::new(ctx.params, ctx.layer_size, ctx.fps(), scale); + let fps = ctx.fps(); + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let buf = host?.self_at(g as f64 / fps, effects)?; + let im = layer_grid(&buf, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let seg = chain.ensure(f, &mut frame, &|| false)?; + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { resample(&seg, w, h) }; + let mp = matte_params(ctx.params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + for (slot, g) in [(&mut prev, f - 1), (&mut next, f + 1)] { + if chain.data.in_span(g) + && let Some(s) = chain.ensure(g, &mut frame, &|| false) + { + *slot = Some(if s.w == w && s.h == h { (*s).clone() } else { resample(&s, w, h) }); + } + } + } + let alpha = final_alpha(img, &seg, prev.as_ref(), next.as_ref(), &mp, chain.refine_radius, scale); + Some((alpha, seg.refine, chain.refine_radius)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + let d = data(ctx.params); + if d.is_empty() || ctx.adjustment { + return buf; + } + let f = frame_of(ctx.time, ctx.fps()); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + if !d.in_span(f) { + return out; + } + let img = Arc::new(layer_grid(&buf, ctx.layer_size)); + let Some((alpha, band, rr)) = matte_for(ctx, f, &img, sc) else { return out }; + let (w, h) = (img.width as usize, img.height as usize); + let mp = matte_params(ctx.params); + let mut rgb = rb::rgb_of(&img); + let map = refine::decontaminate(&mut rgb, &alpha, &band, w, h, &mp, rr * sc, sc); + let view_map = b(ctx.params, "refineEdgeMatte/viewDecontaminationMap"); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + for y in 0..h { + for x in 0..w { + let (bx, by) = (x as i64 + ox, y as i64 + oy); + if bx < 0 || by < 0 || bx >= out.img.width as i64 || by >= out.img.height as i64 { + continue; + } + let i = y * w + x; + let a_in = img.data[i][3]; + let px = if view_map { + let v = map[i]; + [v, v, v, 1.0] + } else { + let a = alpha[i] * a_in; + [rgb[i][0] * a, rgb[i][1] * a, rgb[i][2] * a, a] + }; + out.img.set(bx as u32, by as u32, px); + } + } + out +} + +/// The final matte of the effect at layer frame `f` on the full-resolution layer grid of +/// `input` (the effect's input buffer at scale 1), computing what is missing. For Freeze and +/// the Layer panel overlays. +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option<(Vec, Vec)> { + let chain = Chain::new(params, layer_size, fps, 1.0); + let img = frame(f)?; + let seg = chain.ensure(f, frame, cancel)?; + let mp = matte_params(params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + if chain.data.in_span(f - 1) { + prev = chain.ensure(f - 1, frame, cancel); + } + if chain.data.in_span(f + 1) { + next = chain.ensure(f + 1, frame, cancel); + } + } + let alpha = final_alpha(&img, &seg, prev.as_deref(), next.as_deref(), &mp, chain.refine_radius, 1.0); + Some((alpha, seg.refine.clone())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A host serving a moving red disk over a blue/green background as the effect's input. + struct Seq; + fn frame(t: f64) -> Image { + let fi = (t * 25.0 + 1e-6).floor(); + let c = [40.0 + 2.0 * fi, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let fg = [0.85 + tex, 0.2, 0.1]; + let bg = [0.1, 0.4 + tex, 0.7]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params_with(d: &RotoData) -> Params { + let s = &specs()[0]; + let mut params = Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() }; + params.values.insert(STROKES.into(), Value::Str(d.to_json())); + params + } + + #[test] + fn renders_the_segmented_disk_on_demand() { + let mut d = RotoData::default(); + d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]] }, 20, [0, 24]); + d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]] }, 20, [0, 24]); + let params = params_with(&d); + let host = Seq; + let run = |t: f64| { + let ctx = EffectCtx { + params: ¶ms, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }).img + }; + // Frame 5 (propagated through frames 1–4). + let out = run(5.0 / 25.0 + 0.001); + let c = [50.0, 40.0]; + let mut wrong = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + wrong += 1; + } + } + } + assert!(wrong < 20, "{wrong} wrong pixels"); + // Outside the span: transparent. + let mut d2 = (*data(¶ms)).clone(); + d2.span = [0, 3]; + let p2 = params_with(&d2); + let ctx = EffectCtx { + params: &p2, + time: 0.3, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + let o = crate::apply(&specs()[0], &ctx, Buf { img: frame(0.3), offset: [0.0; 2], scale: 1.0 }).img; + assert!(o.data.iter().all(|p| p[3] == 0.0)); + } +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd78..8ebd78db6 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,6 +525,9 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + if matches!(what.as_str(), "all" | "image" | "memory" | "memoryAndDisk") { + effectcraft_effects::roto::purge(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); Ok(json!({"purged": what})) diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f6..ac4518720 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -28,6 +28,7 @@ mod prop; pub mod puppet; mod query; mod render_queue; +pub mod roto_cmds; mod settings; mod stubs; mod styles; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(roto_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs new file mode 100644 index 000000000..5b9db20fa --- /dev/null +++ b/crates/engine/src/commands/roto_cmds.rs @@ -0,0 +1,424 @@ +//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer +//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. +//! +//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the +//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base +//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze +//! first). Every edit is one undo step. + +use effectcraft_effects::roto::{self as fx, FROZEN, Stroke, StrokeKind}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::Ids; +use effectcraft_project::{ItemId, Layer, LayerId, PropGroup, Uid}; +use effectcraft_track::roto::{RotoData, iou, rle}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::roto::{self as rt, Direction, RotoOptions, RotoTask, VIEWS, is_roto, params_static}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "roto.stroke", + "Roto Brush Stroke", + [], + None, + "{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?}", + has_comp, + stroke + ), + cmd!( + "roto.propagate", + "Propagate Roto Brush", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, to? (layer frame: extend the span), wait?: block until done}", + has_comp, + propagate + ), + cmd!("roto.span", "Set Segmentation Span", [], None, "{layer?, effect?, start?, end? (layer frames)}", has_comp, span), + cmd!("roto.freeze", "Freeze", [], None, "{layer?, effect?, wait?}", has_comp, freeze), + cmd!("roto.unfreeze", "Unfreeze", [], None, "{layer?, effect?}", has_comp, unfreeze), + cmd!("roto.clearStrokes", "Remove Roto Brush Strokes", [], None, "{layer?, effect?, frame? (only this layer frame), kind?}", has_comp, clear_strokes), + cmd!("roto.cancel", "Stop Roto Brush", [], None, "{}", is_running, |s, _| Ok(json!({"stopped": s.stop_roto()}))), + cmd!( + "roto.options", + "Roto Brush Options", + [], + None, + "{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?}", + always, + options + ), + query!( + "roto.status", + "Roto Brush Status", + "{layer?, effect?, frame? (layer frame: also report its matte), matte?: include the matte (RLE, base64), compute?: compute the frame if needed, compareTo?: RLE matte to report the IoU against}", + status + ), + ] +} + +fn is_running(s: &Session) -> std::result::Result<(), String> { + if s.is_roto_running() { Ok(()) } else { Err("no Roto Brush job is running".into()) } +} + +/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), +/// else a selected one, else the first on the selected (or any) layer. +pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fxg) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fxg.groups().find(|g| g.uid == n).or_else(|| fxg.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fxg.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is_roto(g))), + }; + if let Some(g) = g.filter(|g| is_roto(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let g = l.props.find_group(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + Ok((l, g)) +} + +fn data_of(g: &PropGroup) -> RotoData { + fx::data(¶ms_static(g)).as_ref().clone() +} + +fn frozen_of(g: &PropGroup) -> bool { + matches!(g.get(FROZEN).map(|p| &p.value), Some(KV::Str(s)) if !s.is_empty()) +} + +/// Write an instance's data (one undo step). +fn write_data(s: &mut Session, label: &str, cid: ItemId, lid: LayerId, uid: Uid, d: &RotoData) -> Result<()> { + let j = d.to_json(); + s.edit(label, None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(j); + } + Ok(()) + }) +} + +/// Current layer frame of `layer` (or `frame` from the params). +fn frame_p(s: &Session, p: &Value, cid: ItemId, layer: &Layer) -> i64 { + if let Some(f) = f_p(p, "frame") { + return f.round() as i64; + } + let comp = s.project.comp(cid); + let fps = comp.map(|c| c.frame_rate.as_f64()).unwrap_or(30.0); + fx::frame_of(layer.layer_time(s.time_of(cid)).seconds(), fps) +} + +fn points_p(p: &Value, cmd: &str) -> Result> { + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer pixels)"))?; + let mut v = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + v.push([x, y]); + } + if v.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + Ok(v) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.stroke"; + let (cid, lid) = match roto_p(s, p, cmd) { + Ok((c, l, _)) => (c, l), + Err(_) => layer_p(s, p, cmd)?, + }; + let kind = match str_p(p, "kind") { + Some(k) => StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (fg | bg | refine | refineErase)")))?, + None => StrokeKind::Fg, + }; + let points = points_p(p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "the Roto Brush works on footage, solid and composition layers")); + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + let o = &s.state.roto; + let def_r = if matches!(kind, StrokeKind::Refine | StrokeKind::RefineErase) { o.refine_diameter / 2.0 } else { o.diameter / 2.0 }; + let radius = f_p(p, "radius").unwrap_or(def_r).clamp(0.5, 2500.0); + let size = rt::layer_size(&s.project, comp, layer); + // The instance: the requested / first one, else a new one. + let existing = roto_p(s, p, cmd).ok().filter(|(c, l, _)| *c == cid && *l == lid).map(|x| x.2); + if let Some(uid) = existing { + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + } + if existing.is_none() && !kind.is_segmentation() { + return Err(bad(cmd, "draw a foreground stroke with the Roto Brush tool first")); + } + let st = Stroke { kind, frame, radius, points }; + let label = if kind.is_segmentation() { "Roto Brush Stroke" } else { "Refine Edge Stroke" }; + let (uid, d) = s.edit(label, None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let uid = match existing { + Some(u) => u, + None => { + let spec = effectcraft_effects::find(fx::ID).ok_or_else(|| EngineError::Other("Roto Brush effect missing".into()))?; + let n = fxg.groups().filter(|g| is_roto(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let u = g.uid; + fxg.children.push(g.into()); + u + } + }; + let g = fxg.find_group_mut(uid).ok_or_else(|| bad(cmd, "effect gone"))?; + let mut d = data_of(g); + d.add(st, fx::DEFAULT_SPAN, limits); + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(d.to_json()); + } + proj.next_id = proj.next_id.max(next); + Ok((uid, d)) + })?; + if !s.state.selected_props.iter().any(|(l, u)| *l == lid && *u == uid) { + s.state.selected_layers = vec![lid]; + } + Ok(json!({ + "layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius, + "base": d.base, "span": d.span, "strokes": d.strokes.len(), + })) +} + +fn propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.propagate"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + if let Some(to) = f_p(p, "to") { + let to = to.round() as i64; + let (l, g) = group(s, cid, lid, uid)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let to = to.clamp(lim[0], lim[1]); + let before = d.span; + d.span[0] = d.span[0].min(to); + d.span[1] = d.span[1].max(to); + if d.span != before { + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + write_data(s, "Extend Segmentation Span", cid, lid, uid, &d)?; + } + } + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn span(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.span"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (l, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let base = d.base.ok_or_else(|| bad(cmd, "no strokes yet"))?; + let first = d.strokes.iter().map(|s| s.frame).min().unwrap_or(base); + let last = d.strokes.iter().map(|s| s.frame).max().unwrap_or(base); + if let Some(a) = f_p(p, "start") { + d.span[0] = (a.round() as i64).clamp(lim[0], first); + } + if let Some(b) = f_p(p, "end") { + d.span[1] = (b.round() as i64).clamp(last, lim[1]); + } + write_data(s, "Segmentation Span", cid, lid, uid, &d)?; + Ok(json!({"layer": lid.0, "effect": uid, "span": d.span, "base": base})) +} + +fn freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.freeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.unfreeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if !frozen_of(g) { + return Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})); + } + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn clear_strokes(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.clearStrokes"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + let mut d = data_of(g); + let frame = f_p(p, "frame").map(|f| f.round() as i64); + let kind = match str_p(p, "kind") { + Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?), + None => None, + }; + let n0 = d.strokes.len(); + d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k))); + if d.strokes.iter().all(|s| !s.kind.is_segmentation()) { + d.base = None; + } else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) { + // The base moves to the earliest remaining stroked frame. + d.base = d.strokes.iter().filter(|s| s.kind.is_segmentation()).map(|s| s.frame).min(); + } + let removed = n0 - d.strokes.len(); + if removed > 0 { + write_data(s, "Remove Roto Brush Strokes", cid, lid, uid, &d)?; + } + Ok(json!({"layer": lid.0, "effect": uid, "removed": removed, "strokes": d.strokes.len(), "base": d.base})) +} + +fn options(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.options"; + let mut o: RotoOptions = s.state.roto.clone(); + for (k, f) in [("diameter", &mut o.diameter as &mut f64), ("refineDiameter", &mut o.refine_diameter), ("overlayOpacity", &mut o.overlay_opacity)] { + if let Some(v) = f_p(p, k) { + *f = v; + } + } + o.diameter = o.diameter.clamp(1.0, 5000.0); + o.refine_diameter = o.refine_diameter.clamp(1.0, 5000.0); + o.overlay_opacity = o.overlay_opacity.clamp(0.0, 100.0); + if let Some(v) = str_p(p, "view") { + let v = VIEWS.iter().find(|x| x.eq_ignore_ascii_case(v)).ok_or_else(|| bad(cmd, format!("unknown view `{v}` ({})", VIEWS.join(" | "))))?; + o.view = v.to_string(); + } + for (k, c) in [("overlayColor", &mut o.overlay_color), ("boundaryColor", &mut o.boundary_color)] { + if let Some(a) = p.get(k).and_then(Value::as_array) { + if a.len() < 3 { + return Err(bad(cmd, format!("`{k}` is [r, g, b] (0–1)"))); + } + for (i, v) in a.iter().take(3).enumerate() { + c[i] = v.as_f64().unwrap_or(0.0).clamp(0.0, 1.0); + } + } + } + if let Some(b) = b_p(p, "autoPropagate") { + o.auto_propagate = b; + } + s.state.roto = o.clone(); + Ok(serde_json::to_value(o).unwrap_or(Value::Null)) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let mut out = + json!({"running": s.is_roto_running(), "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner()), "options": s.state.roto}); + let Ok((cid, lid, uid)) = roto_p(s, p, "roto.status") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let (layer, g) = group(s, cid, lid, uid)?; + let (layer, g) = (layer.clone(), g.clone()); + let d = data_of(&g); + let computed = s.roto_computed(cid, lid, uid); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["base"] = json!(d.base); + out["span"] = json!(d.span); + out["strokes"] = json!(d.strokes.len()); + out["strokeFrames"] = json!(d.stroke_frames()); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + out["frozen"] = json!(frozen_of(&g)); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["limits"] = json!(rt::frame_limits(&comp, &layer)); + let frame = frame_p(s, p, cid, &layer); + out["frame"] = json!(frame); + let want_matte = b_p(p, "matte").unwrap_or(false); + let compute = b_p(p, "compute").unwrap_or(want_matte); + if (p.get("frame").is_some() || want_matte) && d.in_span(frame) { + let size = rt::layer_size(&s.project, &comp, &layer); + let fps = comp.frame_rate.as_f64(); + let ps = params_static(&g); + let chain = fx::Chain::new(&ps, size, fps, 1.0); + let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) { + Some(sg) => Some(sg), + None if compute => { + let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0); + let mut src = rt::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, cid, &comp, &layer, index, size); + chain.ensure(frame, &mut |f| src.get(f), &|| false) + } + None => None, + }; + if let Some(sg) = seg { + out["area"] = json!(sg.area()); + out["centroid"] = json!(sg.centroid()); + out["size"] = json!([sg.w, sg.h]); + out["refineArea"] = json!(sg.refine.iter().filter(|v| **v != 0).count()); + if want_matte { + out["matte"] = json!(rle::encode(&sg.matte)); + } + if let Some(gt) = p.get("compareTo").and_then(Value::as_str).and_then(rle::decode) { + out["iou"] = json!(iou(&sg.matte, >)); + } + } + } + Ok(out) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488ab..fa2e6fe47 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -17,6 +17,7 @@ pub mod mask_track; pub mod menus; pub mod prefs; pub mod render_queue; +pub mod roto; mod session_settings; pub mod shortcuts; pub mod tracking; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -176,6 +178,9 @@ pub struct EditorState { /// Paint and Brushes panel options (Brush, Clone Stamp and Eraser tools). #[serde(default)] pub paint: commands::paint::PaintOptions, + /// Roto Brush and Refine Edge tool options. + #[serde(default)] + pub roto: roto::RotoOptions, /// Puppet tool options for new meshes. #[serde(default)] pub puppet: commands::puppet::PuppetOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) Roto Brush propagation / Freeze. + pub roto_job: Option, + /// Roto Brush instances edited since their last propagation: (comp, layer, effect uid). + pub roto_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + roto_job: None, + roto_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // Roto Brush Input Keys follow the frames; edited instances propagate again. + for r in roto::sync(&before, &mut p) { + if !self.roto_pending.contains(&r) { + self.roto_pending.push(r); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_roto(); + self.roto_job = None; + self.roto_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -615,3 +635,5 @@ pub fn text_families() -> Vec { } #[cfg(test)] mod tests_paint; +#[cfg(test)] +mod tests_roto; diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs new file mode 100644 index 000000000..5b2f635e0 --- /dev/null +++ b/crates/engine/src/roto.rs @@ -0,0 +1,569 @@ +//! Roto Brush & Refine Edge runtime: propagation and Freeze jobs, Input Key maintenance and the +//! tool options. +//! +//! Segmentations are derived data in `effectcraft_effects::roto`'s content-keyed cache (see that +//! module). `roto.propagate` fills it at full resolution in the background (or blocking with +//! `wait`), frame by frame outward from the base frame, rendering the effect's input +//! (`Renderer::layer_input`, source → masks → the effects above it). `roto.freeze` runs the same +//! pass and stores the final mattes in the effect (one undo step). [`sync`], run after every +//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made +//! of (the layer's source and timing, its masks and the effects above it) and drops frozen +//! mattes made from other frames. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; +use effectcraft_effects::{Params, flatten_params}; +use effectcraft_keyframe::Value; +use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; + +use crate::tracking::lock; +use crate::{Event, Session}; + +/// Roto Brush / Refine Edge tool options (serde: `roto.options`). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoOptions { + /// Roto Brush tip diameter (layer pixels). + pub diameter: f64, + /// Refine Edge tip diameter (layer pixels). + pub refine_diameter: f64, + /// Layer panel view: `alphaBoundary` (pink outline), `alpha`, `alphaOverlay`, `none`. + pub view: String, + /// Alpha Overlay colour (straight RGB 0–1) and opacity (percent). + pub overlay_color: [f64; 3], + pub overlay_opacity: f64, + /// Alpha Boundary colour. + pub boundary_color: [f64; 3], + /// Start propagating in the background after each stroke. + pub auto_propagate: bool, +} + +impl Default for RotoOptions { + fn default() -> Self { + RotoOptions { + diameter: 20.0, + refine_diameter: 10.0, + view: "alphaBoundary".into(), + overlay_color: [1.0, 0.0, 0.0], + overlay_opacity: 50.0, + boundary_color: [1.0, 0.25, 0.75], + auto_propagate: true, + } + } +} + +pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"]; + +/// What a job does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum RotoTask { + Propagate, + Freeze, +} + +/// Which way to propagate. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Direction { + Forward, + Backward, + #[default] + Both, +} + +impl Direction { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "forward" | "fwd" => Direction::Forward, + "backward" | "back" => Direction::Backward, + "both" | "all" => Direction::Both, + _ => return None, + }) + } +} + +/// Live job progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RotoProgress { + pub task: Option, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, +} + +impl RotoProgress { + pub fn banner(&self) -> String { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + match self.task { + Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + _ => format!("Propagating Roto Brush: {pc}%"), + } + } +} + +#[derive(Default)] +pub struct RotoShared { + pub cancel: AtomicBool, + pub state: Mutex, + frozen: Mutex>, +} + +pub struct RotoJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + pub task: RotoTask, + /// The strokes the job was started with (a Freeze result is dropped when they changed). + strokes: String, +} + +impl RotoJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> RotoProgress { + lock(&self.shared.state).clone() + } +} + +pub(crate) fn is_roto(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.get(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +fn set_str(g: &mut PropGroup, m: &str, v: String) { + if let Some(p) = g.get_mut(m) { + p.value = Value::Str(v); + p.keys.clear(); + } +} + +fn fnv(s: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +/// What the effect's input frames are made of: the layer's source and timing, its masks and the +/// effects above the Roto Brush. +pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { + let mut s = crate::warp::signature(p, comp, layer); + if let Some(m) = layer.masks() { + s.push_str(&format!("{m:?}")); + } + if let Some(fx) = layer.effects() { + for g in fx.groups() { + if g.uid == uid { + break; + } + s.push_str(&format!("{g:?}")); + } + } + format!("{:016x}", fnv(&s)) +} + +/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. +/// Returns the instances whose segmentation changed (to propagate again). +pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut set = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fxg) = l.effects() else { continue }; + for g in fxg.groups().filter(|g| is_roto(g)) { + let key = input_key(after, *cid, l, g.uid); + let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); + if str_of(g, INPUT_KEY) != key { + set.push((*cid, l.id, g.uid, key)); + todo.push((*cid, l.id, g.uid)); + } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, key) in set { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + let had = !str_of(g, INPUT_KEY).is_empty(); + set_str(g, INPUT_KEY, key); + if had { + set_str(g, FROZEN, String::new()); + } + } + } + todo.retain(|(c, l, u)| { + after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + }); + todo +} + +/// Parameters of an instance with static values (strokes and keys are never animated). +pub(crate) fn params_static(g: &PropGroup) -> Params { + flatten_params(g, &mut |p| p.value.clone()) +} + +/// Evaluated parameters of an instance at comp time `t`. +pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup, t: Tick, expr: Option<&dyn ExprHost>) -> Params { + match p.comp(cid) { + Some(c) => { + let ctx = EvalCtx { expr, ..EvalCtx::new(p, cid, c, t) }; + flatten_params(g, &mut |pr| ctx.value(layer, pr)) + } + None => params_static(g), + } +} + +/// The layer's source size (layer pixels). +pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] { + let (w, h) = effectcraft_render::source_size(p, layer); + if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] } +} + +/// First and last layer frame of the layer inside the comp (inclusive). +pub fn frame_limits(comp: &effectcraft_project::Comp, layer: &Layer) -> [i64; 2] { + let fps = comp.frame_rate.as_f64().max(1e-6); + let lo = layer.layer_time(layer.in_point.max(Tick::ZERO)).seconds(); + let hi = layer.layer_time(layer.out_point.min(comp.duration)).seconds(); + let a = fx::frame_of(lo, fps); + let b = (fx::frame_of(hi, fps) - 1).max(a); + [a, b] +} + +/// Comp time of layer frame `f`. +pub(crate) fn comp_time_of(comp: &effectcraft_project::Comp, layer: &Layer, f: i64) -> Tick { + layer.comp_time(comp.frame_rate.tick_of(f)) +} + +/// What the worker needs, detached from the session. +struct Work { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + params: Params, + size: [f64; 2], + fps: f64, + frames: Vec, + task: RotoTask, +} + +/// Renders the effect's input frames for a worker, keeping the last few. +pub(crate) struct FrameSource<'a> { + r: Renderer<'a>, + ctx0: EvalCtx<'a>, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + memo: HashMap>, +} + +impl<'a> FrameSource<'a> { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + project: &'a Project, + footage: &'a dyn FootageSource, + expr: Option<&'a dyn ExprHost>, + cache: &'a LayerCache, + cid: ItemId, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + ) -> FrameSource<'a> { + let mut r = Renderer::new(project, footage, RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = expr; + r.cache = Some(cache); + let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) }; + FrameSource { r, ctx0, comp, layer, index, size, memo: HashMap::new() } + } + + pub(crate) fn get(&mut self, f: i64) -> Option> { + if let Some(i) = self.memo.get(&f) { + return Some(i.clone()); + } + let t = comp_time_of(self.comp, self.layer, f); + let buf = self.r.layer_input(&self.ctx0.at(t), self.layer, self.index)?; + let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + effectcraft_effects::Buf { + img: effectcraft_raster::resample(&buf.img, w, h), + offset: [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale], + scale: 1.0, + } + } else { + (*buf).clone() + }; + let img = Arc::new(fx::layer_grid(&buf, self.size)); + if self.memo.len() > 4 { + self.memo.clear(); + } + self.memo.insert(f, img.clone()); + Some(img) + } +} + +fn run_work(w: Work, shared: &RotoShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut src = FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size); + { + let mut s = lock(&shared.state); + s.task = Some(w.task); + s.total = w.frames.len() as u64; + } + let cancel = || shared.cancel.load(Ordering::Relaxed); + let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); + let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + for (k, f) in w.frames.iter().enumerate() { + if cancel() { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + return; + } + let mut get = |g: i64| src.get(g); + let ok = match w.task { + RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), + RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + Some((a, b)) => { + if let Some(img) = src.get(*f) { + frozen.size = [img.width as usize, img.height as usize]; + } + frozen.frames.insert(*f, fx::freeze_frame(&a, &b)); + true + } + None => false, + }, + }; + if !ok && !cancel() { + return fail(&format!("frame {f} could not be segmented")); + } + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + if w.task == RotoTask::Freeze { + *lock(&shared.frozen) = Some(frozen); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// Frames to compute, outward from the base frame. +pub(crate) fn job_frames(d: &effectcraft_track::roto::RotoData, dir: Direction) -> Vec { + let Some(base) = d.base else { return vec![] }; + let mut v = vec![base]; + if dir != Direction::Backward { + v.extend(base + 1..=d.span[1]); + } + if dir != Direction::Forward { + v.extend((d.span[0]..base).rev()); + } + v +} + +impl Session { + pub fn is_roto_running(&self) -> bool { + self.roto_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + pub fn roto_progress(&self) -> Option { + self.roto_job.as_ref().map(|j| j.progress()) + } + + /// (comp, layer, effect uid, task) of the running job. + pub fn roto_target(&self) -> Option<(ItemId, LayerId, Uid, RotoTask)> { + self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)) + } + + pub fn stop_roto(&mut self) -> bool { + match &self.roto_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start propagating (or freezing) Roto Brush `effect` on `layer`. `wait`: block until done + /// (wasm32 always does). Returns the number of frames. + pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result { + if self.is_roto_running() { + if !wait { + return Err("a Roto Brush job is already running".into()); + } + self.stop_roto(); + if let Some(mut j) = self.roto_job.take() + && let Some(t) = j.thread.take() + { + let _ = t.join(); + } + } + self.poll_roto_job(); + self.roto_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fxg = l.effects().ok_or("the layer has no effects")?; + let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let size = layer_size(&self.project, c, l); + let fps = c.frame_rate.as_f64(); + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let n = frames.len(); + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + params, + size, + fps, + frames, + task, + }; + let strokes = str_of(g, fx::STROKES); + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(RotoShared::default()); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("roto".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_roto_job(); + Ok(n) + } + + /// Finish a done job: Freeze writes the frozen mattes (one undo step). + fn poll_roto_job(&mut self) -> bool { + let Some(job) = &self.roto_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.roto_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + if let Some(e) = &st.error { + self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.bump(); + return true; + } + if st.cancelled { + self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.bump(); + return true; + } + match job.task { + RotoTask::Propagate => { + self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + // Rendered frames depend on the cache: redraw. + self.bump(); + } + RotoTask::Freeze => { + let Some(fc) = lock(&job.shared.frozen).take() else { return true }; + let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let r = self.edit("Freeze", None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; + if str_of(g, fx::STROKES) != strokes { + return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + } + set_str(g, FROZEN, fc.to_json()); + Ok(()) + }); + let msg = match r { + Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("Roto Brush: {e}"), + }; + self.events.push(Event::Toast { message: msg, error: false }); + } + } + true + } + + /// Frontends call this every frame: finishes jobs and (with `auto`) propagates edited + /// instances in the background. + pub fn poll_roto(&mut self, auto: bool) -> bool { + let changed = self.poll_roto_job(); + if auto && self.roto_job.is_none() && self.state.roto.auto_propagate { + while let Some((c, l, e)) = self.roto_pending.first().copied() { + self.roto_pending.remove(0); + if self.start_roto(c, l, e, RotoTask::Propagate, Direction::Both, false).is_ok() { + return true; + } + } + } + changed + } + + /// The computed frames of an instance (full resolution), by layer frame. + pub fn roto_computed(&self, comp: ItemId, layer: LayerId, effect: Uid) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(c) = self.project.comp(comp) else { return out }; + let Some(l) = c.layer(layer) else { return out }; + let Some(g) = l.props.find_group(effect) else { return out }; + let ps = params_static(g); + let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); + for f in ch.keys.keys() { + out.insert(*f, ch.is_cached(*f, 1.0)); + } + out + } +} diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs new file mode 100644 index 000000000..794036269 --- /dev/null +++ b/crates/engine/src/tests_roto.rs @@ -0,0 +1,288 @@ +//! Roto Brush & Refine Edge end to end through the commands, on synthetic footage: a textured +//! disk moving over a textured background. + +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::RenderOpts; +use effectcraft_track::roto::rle; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup}; + +const R: f64 = 40.0; + +fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +/// Disk centre on frame `f`. +fn center(f: u32) -> [f64; 2] { + let t = f as f64; + [90.0 + 4.0 * t, 110.0 + 1.5 * t + 5.0 * (t * 0.3).sin()] +} + +/// Disk centre with a jump at frame 8 (propagation loses it there). +fn jumping(f: u32) -> [f64; 2] { + if f < 8 { center(f) } else { [230.0 + 2.0 * (f - 8) as f64, 150.0] } +} + +fn draw(c: [f64; 2]) -> Image { + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let n = noise(p[0] - c[0], p[1] - c[1], 11); + let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32]; + let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9)); + let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + } + } + img +} + +fn gt(c: [f64; 2]) -> String { + let m: Vec = (0..W * H).map(|i| (((i % W) as f64 + 0.5 - c[0]).hypot((i / W) as f64 + 0.5 - c[1]) <= R) as u8).collect(); + rle::encode(&m) +} + +fn line(a: [f64; 2], b: [f64; 2]) -> Value { + json!((0..=8).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 8.0, a[1] + (b[1] - a[1]) * k as f64 / 8.0]).collect::>()) +} + +/// A session with the clip (the Track solid removed) and foreground / background strokes on +/// frame 0. `name`: the footage file name (segmentations are cached process-wide by content key, +/// which includes the footage's identity: different clips need different names). +fn painted(path: fn(u32) -> [f64; 2], name: &str) -> (Session, LayerId, u64) { + let (mut s, clip, solid) = setup(move |f| draw(path(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = name.into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = path(0); + let r = + s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": line([c[0] - 22.0, c[1] - 8.0], [c[0] + 22.0, c[1] + 8.0]), "radius": 6})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert_eq!(r["base"], 0); + assert_eq!(r["span"], json!([0, 20])); + for (a, b) in [([c[0] - 60.0, c[1] - 58.0], [c[0] + 70.0, c[1] - 58.0]), ([c[0] + 70.0, c[1] + 58.0], [c[0] - 60.0, c[1] + 58.0])] { + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line(a, b), "radius": 6})).unwrap(); + } + (s, clip, uid) +} + +/// Drop an instance's derived segmentations from the process-wide cache. +fn forget(s: &Session, clip: LayerId, uid: u64) { + let comp = s.active_comp().unwrap(); + let l = comp.layer(clip).unwrap(); + let g = l.props.find_group(uid).unwrap(); + let size = crate::roto::layer_size(&s.project, comp, l); + let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0); + effectcraft_effects::roto::forget(ch.keys.values().copied()); +} + +fn iou_at(s: &mut Session, clip: LayerId, f: u32, c: [f64; 2]) -> f64 { + let st = s.execute("roto.status", json!({"layer": clip.0, "frame": f, "compute": true, "compareTo": gt(c)})).unwrap(); + st["iou"].as_f64().unwrap_or_else(|| panic!("no matte on frame {f}: {st}")) +} + +fn alpha_at(s: &Session, f: u32) -> Vec { + let img = s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts { scale: 1.0, ..Default::default() }); + img.data.iter().map(|p| p[3]).collect() +} + +#[test] +fn roto_brush_segments_and_propagates() { + let (mut s, clip, uid) = painted(center, "disk-a.mov"); + let l = s.active_comp().unwrap().layer(clip).unwrap().clone(); + assert_eq!(l.effects().unwrap().groups().filter(|g| crate::roto::is_roto(g)).count(), 1, "one effect for all strokes"); + let j0 = iou_at(&mut s, clip, 0, center(0)); + assert!(j0 > 0.95, "base frame IoU {j0}"); + // Propagate the span (blocking). + let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap(); + assert_eq!(r["frames"], 21, "{r}"); + assert_eq!(r["status"]["computed"].as_array().unwrap().len(), 21); + let mut worst: f64 = 1.0; + for f in 0..=20 { + worst = worst.min(iou_at(&mut s, clip, f, center(f))); + } + assert!(worst > 0.9, "worst IoU over the span {worst}"); + // The render applies the matte; outside the span the layer is transparent. + let a = alpha_at(&s, 12); + let c = center(12); + let mut wrong = 0; + for (i, v) in a.iter().enumerate() { + let d = (((i as u32 % W) as f64 + 0.5 - c[0]).hypot((i as u32 / W) as f64 + 0.5 - c[1])) - R; + if (d < -3.0 && *v < 0.5) || (d > 3.0 && *v > 0.5) { + wrong += 1; + } + } + assert!(wrong < 150, "{wrong} wrong pixels in the render"); + assert!(alpha_at(&s, 23).iter().all(|v| *v == 0.0)); + // Extending the span propagates further. + let r = s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "to": 24, "wait": true})).unwrap(); + assert_eq!(r["status"]["span"], json!([0, 24])); + assert!(iou_at(&mut s, clip, 24, center(24)) > 0.9); + // Undo / redo of strokes and the span. + let n = |s: &Session| { + let st = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().get(effectcraft_effects::roto::STROKES).unwrap().value.clone(); + match st { + effectcraft_keyframe::Value::Str(j) => effectcraft_track::roto::RotoData::from_json(&j), + _ => panic!(), + } + }; + assert_eq!(n(&s).span, [0, 24]); + assert!(s.undo()); + assert_eq!(n(&s).span, [0, 20]); + assert_eq!(n(&s).strokes.len(), 3); + assert!(s.undo()); + assert_eq!(n(&s).strokes.len(), 2); + assert!(s.redo()); + assert!(s.redo()); + assert_eq!(n(&s).strokes.len(), 3); + assert_eq!(n(&s).span, [0, 24]); + // Serde keeps the strokes; the reopened project renders the same matte. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + assert_eq!( + crate::roto::params_static(&g).get("strokes"), + crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()).get("strokes") + ); + // Bad parameters are reported. + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "sideways", "points": [[1, 1]]})).is_err()); + assert!(s.execute_checked("roto.propagate", json!({"layer": clip.0, "bogus": 1})).is_err()); + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": [[1, 1]], "frame": 99})).is_err()); +} + +#[test] +fn correction_strokes_restart_propagation() { + let (mut s, clip, _) = painted(jumping, "disk-jump.mov"); + s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); + assert!(iou_at(&mut s, clip, 6, jumping(6)) > 0.9); + let bad = iou_at(&mut s, clip, 8, jumping(8)); + assert!(bad < 0.5, "the jump should break propagation ({bad})"); + // Correct frame 8: paint the disk in and its old place out. + let (c, old) = (jumping(8), center(8)); + s.execute("roto.stroke", json!({"layer": clip.0, "frame": 8, "points": line([c[0] - 20.0, c[1]], [c[0] + 20.0, c[1]]), "radius": 6})).unwrap(); + s.execute( + "roto.stroke", + json!({"layer": clip.0, "frame": 8, "kind": "bg", "points": line([old[0] - 15.0, old[1] - 15.0], [old[0] + 15.0, old[1] + 15.0]), "radius": 8}), + ) + .unwrap(); + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + let computed: Vec = serde_json::from_value(st["computed"].clone()).unwrap(); + assert!(computed.contains(&7) && !computed.contains(&8) && !computed.contains(&9), "frames from the correction on are recomputed: {computed:?}"); + assert_eq!(st["strokeFrames"], json!([0, 8])); + s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "wait": true})).unwrap(); + for f in 8..=20 { + let j = iou_at(&mut s, clip, f, jumping(f)); + assert!(j > 0.9, "frame {f}: IoU {j}"); + } + // Clearing the correction brings the bad frame back. + let r = s.execute("roto.clearStrokes", json!({"layer": clip.0, "frame": 8})).unwrap(); + assert_eq!(r["removed"], 2); + assert!(iou_at(&mut s, clip, 8, jumping(8)) < 0.5); +} + +#[test] +fn freeze_caches_the_mattes() { + let (mut s, clip, uid) = painted(center, "disk-freeze.mov"); + // A refine band along the edge on the base frame. + let c = center(0); + let ring: Vec<[f64; 2]> = (0..=40).map(|k| k as f64 / 40.0 * std::f64::consts::TAU).map(|a| [c[0] + R * a.cos(), c[1] + R * a.sin()]).collect(); + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "refine", "points": ring, "radius": 5})).unwrap(); + s.execute("roto.span", json!({"layer": clip.0, "end": 3})).unwrap(); + let live: Vec> = (0..=3).map(|f| alpha_at(&s, f)).collect(); + // Freeze in the background and wait for it. + let r = s.execute_checked("roto.freeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 4); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["frozen"], true, "{st}"); + // Frozen renders match the live ones (8-bit alpha), even without the derived segmentations. + forget(&s, clip, uid); + s.layer_cache = Default::default(); + for (f, want) in live.iter().enumerate() { + let got = alpha_at(&s, f as u32); + let worst = got.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(worst <= 1.0 / 255.0 + 1e-4, "frame {f}: {worst}"); + } + assert!(st["computed"].as_array().unwrap().len() >= 4); + // Frozen effects refuse strokes; Unfreeze is undoable. + assert!(s.execute("roto.stroke", json!({"layer": clip.0, "points": [[10, 10]]})).is_err()); + s.execute("roto.unfreeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], false); + assert!(s.undo()); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], true); + // Changing the source frames (the layer's timing) drops the frozen mattes. + let cid = s.active_comp_id().unwrap(); + s.edit("shift", None, |p, _| { + let l = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap(); + l.start_time = frame_time(1); + Ok(()) + }) + .unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0, "effect": uid})).unwrap()["frozen"], false); +} + +#[test] +fn propagation_is_deterministic_and_runs_in_background() { + let (mut s, clip, uid) = painted(center, "disk-bg.mov"); + s.execute("roto.span", json!({"layer": clip.0, "end": 5})).unwrap(); + s.execute("roto.propagate", json!({"layer": clip.0})).unwrap(); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["computed"], json!([0, 1, 2, 3, 4, 5]), "{st}"); + let a = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + // Recomputing from scratch gives the same matte. + forget(&s, clip, uid); + let b = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + assert!(a.is_string()); + assert_eq!(a, b); + // Tool options. + let o = s.execute_checked("roto.options", json!({"diameter": 30, "view": "alphaOverlay"})).unwrap(); + assert_eq!(o["view"], "alphaOverlay"); + assert_eq!(s.state.roto.diameter, 30.0); + assert!(s.execute("roto.options", json!({"view": "sideways"})).is_err()); + // The default radius follows the tool diameter. + let r = s.execute("roto.stroke", json!({"layer": clip.0, "points": [[90, 110]]})).unwrap(); + assert_eq!(r["radius"], 15.0); +} diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92b..70e08ae06 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5f..629097cf0 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -22,6 +22,7 @@ pub mod fit; pub mod klt; pub mod mask; pub mod plane; +pub mod roto; pub mod solve; pub mod stabilize; diff --git a/crates/track/src/roto/gmm.rs b/crates/track/src/roto/gmm.rs new file mode 100644 index 000000000..c0fe95b05 --- /dev/null +++ b/crates/track/src/roto/gmm.rs @@ -0,0 +1,228 @@ +//! Gaussian mixture colour models (full 3 × 3 covariances), as used by GrabCut (C. Rother, +//! V. Kolmogorov and A. Blake, "GrabCut: Interactive Foreground Extraction using Iterated Graph +//! Cuts", SIGGRAPH 2004). +//! +//! Fitting is deterministic: components are initialised by recursive splitting of the cluster +//! with the largest variance along its principal axis (Orchard & Bouman, "Color Quantization of +//! Images", IEEE TSP 1991), then refined by a few hard-assignment EM iterations. + +use serde::{Deserialize, Serialize}; + +/// One Gaussian: weight, mean, inverse covariance and log normaliser. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Gauss { + pub weight: f64, + pub mean: [f64; 3], + pub inv: [[f64; 3]; 3], + /// −log(weight) + ½·log(det Σ) (the 2π term is dropped: it cancels between models). + pub log_norm: f64, +} + +/// A colour model. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Gmm { + pub comps: Vec, +} + +/// Regularisation added to every covariance's diagonal (colour noise floor). +const REG: f64 = 2e-4; + +fn det3(m: &[[f64; 3]; 3]) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} + +fn inv3(m: &[[f64; 3]; 3]) -> Option<[[f64; 3]; 3]> { + let d = det3(m); + if d.abs() < 1e-30 { + return None; + } + let mut r = [[0.0; 3]; 3]; + r[0][0] = (m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d; + r[0][1] = (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d; + r[0][2] = (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d; + r[1][0] = (m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d; + r[1][1] = (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d; + r[1][2] = (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d; + r[2][0] = (m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d; + r[2][1] = (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d; + r[2][2] = (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d; + Some(r) +} + +/// Sufficient statistics of a cluster. +#[derive(Clone, Copy, Default)] +struct Stats { + n: f64, + s: [f64; 3], + ss: [[f64; 3]; 3], +} + +impl Stats { + fn add(&mut self, c: [f32; 3]) { + let c = [c[0] as f64, c[1] as f64, c[2] as f64]; + self.n += 1.0; + for i in 0..3 { + self.s[i] += c[i]; + for j in 0..3 { + self.ss[i][j] += c[i] * c[j]; + } + } + } + fn mean(&self) -> [f64; 3] { + let n = self.n.max(1.0); + [self.s[0] / n, self.s[1] / n, self.s[2] / n] + } + fn cov(&self) -> [[f64; 3]; 3] { + let n = self.n.max(1.0); + let m = self.mean(); + let mut c = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + c[i][j] = self.ss[i][j] / n - m[i] * m[j]; + } + c[i][i] += REG; + } + c + } +} + +/// Principal eigenvector and eigenvalue of a symmetric 3 × 3 matrix (power iteration). +fn principal(c: &[[f64; 3]; 3]) -> ([f64; 3], f64) { + let mut v = [0.577, 0.577, 0.577]; + let mut lambda = 0.0; + for _ in 0..40 { + let w = [ + c[0][0] * v[0] + c[0][1] * v[1] + c[0][2] * v[2], + c[1][0] * v[0] + c[1][1] * v[1] + c[1][2] * v[2], + c[2][0] * v[0] + c[2][1] * v[1] + c[2][2] * v[2], + ]; + let n = (w[0] * w[0] + w[1] * w[1] + w[2] * w[2]).sqrt(); + if n < 1e-30 { + break; + } + lambda = n; + v = [w[0] / n, w[1] / n, w[2] / n]; + } + (v, lambda) +} + +impl Gmm { + /// Fit `k` components to `samples` (at most `max_samples` are used, taken at a fixed stride). + pub fn fit(samples: &[[f32; 3]], k: usize, max_samples: usize) -> Gmm { + if samples.is_empty() { + return Gmm::default(); + } + let stride = samples.len().div_ceil(max_samples.max(1)).max(1); + let xs: Vec<[f32; 3]> = samples.iter().step_by(stride).copied().collect(); + // Orchard–Bouman splitting. + let mut label = vec![0usize; xs.len()]; + let mut ncl = 1; + while ncl < k { + let mut st = vec![Stats::default(); ncl]; + for (x, l) in xs.iter().zip(&label) { + st[*l].add(*x); + } + let (mut best, mut best_l, mut axis) = (usize::MAX, 0.0, [0.0; 3]); + for (c, s) in st.iter().enumerate() { + if s.n < 2.0 { + continue; + } + let (v, l) = principal(&s.cov()); + if l > best_l { + best = c; + best_l = l; + axis = v; + } + } + if best == usize::MAX || best_l <= REG * 1.01 { + break; + } + let m = st[best].mean(); + let thr = axis[0] * m[0] + axis[1] * m[1] + axis[2] * m[2]; + for (x, l) in xs.iter().zip(label.iter_mut()) { + if *l == best && axis[0] * x[0] as f64 + axis[1] * x[1] as f64 + axis[2] * x[2] as f64 > thr { + *l = ncl; + } + } + ncl += 1; + } + let mut g = Gmm::from_labels(&xs, &label, ncl); + // Hard EM. + for _ in 0..3 { + for (x, l) in xs.iter().zip(label.iter_mut()) { + *l = g.best(*x); + } + let ng = Gmm::from_labels(&xs, &label, g.comps.len()); + if ng.comps.is_empty() { + break; + } + g = ng; + } + g + } + + fn from_labels(xs: &[[f32; 3]], label: &[usize], k: usize) -> Gmm { + let mut st = vec![Stats::default(); k]; + for (x, l) in xs.iter().zip(label) { + st[*l].add(*x); + } + let total: f64 = st.iter().map(|s| s.n).sum::().max(1.0); + let comps = st + .iter() + .filter(|s| s.n >= 1.0) + .filter_map(|s| { + let c = s.cov(); + let inv = inv3(&c)?; + let det = det3(&c).max(1e-300); + let weight = s.n / total; + Some(Gauss { weight, mean: s.mean(), inv, log_norm: -weight.ln() + 0.5 * det.ln() }) + }) + .collect(); + Gmm { comps } + } + + #[inline] + fn comp_cost(g: &Gauss, c: [f32; 3]) -> f64 { + let d = [c[0] as f64 - g.mean[0], c[1] as f64 - g.mean[1], c[2] as f64 - g.mean[2]]; + let mut q = 0.0; + for i in 0..3 { + q += d[i] * (g.inv[i][0] * d[0] + g.inv[i][1] * d[1] + g.inv[i][2] * d[2]); + } + g.log_norm + 0.5 * q + } + + /// The most likely component of colour `c`. + pub fn best(&self, c: [f32; 3]) -> usize { + let mut b = 0; + let mut bv = f64::INFINITY; + for (i, g) in self.comps.iter().enumerate() { + let v = Self::comp_cost(g, c); + if v < bv { + bv = v; + b = i; + } + } + b + } + + /// −log p(c) (up to a constant shared by all models), clamped to a finite range. + pub fn cost(&self, c: [f32; 3]) -> f64 { + if self.comps.is_empty() { + return 10.0; + } + // log-sum-exp over components. + let mut m = f64::INFINITY; + let mut costs = [0.0f64; 16]; + let n = self.comps.len().min(16); + for (i, g) in self.comps.iter().take(n).enumerate() { + costs[i] = Self::comp_cost(g, c); + m = m.min(costs[i]); + } + let s: f64 = costs[..n].iter().map(|v| (m - v).exp()).sum(); + (m - s.ln()).clamp(-20.0, 60.0) + } + + pub fn is_empty(&self) -> bool { + self.comps.is_empty() + } +} diff --git a/crates/track/src/roto/matting.rs b/crates/track/src/roto/matting.rs new file mode 100644 index 000000000..439d9c43e --- /dev/null +++ b/crates/track/src/roto/matting.rs @@ -0,0 +1,465 @@ +//! Matte utilities: exact Euclidean distance transforms (Felzenszwalb & Huttenlocher, +//! "Distance Transforms of Sampled Functions", Theory of Computing 8, 2012), box and Gaussian +//! filters, the colour **guided filter** (K. He, J. Sun and X. Tang, "Guided Image Filtering", +//! ECCV 2010 / IEEE PAMI 2013) used for edge matting, foreground colour estimation +//! (decontamination, from the compositing equation I = αF + (1 − α)B with B and F estimated from +//! nearby pure background / foreground pixels), and the Refine Edge matte adjustments. + +use rayon::prelude::*; + +const BIG: f32 = 1e12; + +/// 1-D squared distance transform of `f` (lower envelope of parabolas). +fn dt1(f: &[f32], d: &mut [f32], v: &mut [usize], z: &mut [f64]) { + let n = f.len(); + if n == 0 { + return; + } + let ff = |q: usize| f[q] as f64 + (q * q) as f64; + let mut k = 0usize; + v[0] = 0; + z[0] = f64::NEG_INFINITY; + z[1] = f64::INFINITY; + for q in 1..n { + let mut s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + while s <= z[k] { + if k == 0 { + break; + } + k -= 1; + s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + } + k += 1; + v[k] = q; + z[k] = s; + z[k + 1] = f64::INFINITY; + } + k = 0; + for (q, dq) in d.iter_mut().enumerate().take(n) { + while z[k + 1] < q as f64 { + k += 1; + } + let p = v[k]; + let dd = q as f64 - p as f64; + *dq = (dd * dd + f[p] as f64) as f32; + } +} + +/// Euclidean distance from every pixel to the nearest pixel where `seed` is true (BIG.sqrt() +/// when there is none). +pub fn distance(seed: &[bool], w: usize, h: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + let mut g: Vec = seed.iter().map(|s| if *s { 0.0 } else { BIG }).collect(); + // Columns. + let cols: Vec> = (0..w) + .into_par_iter() + .map(|x| { + let f: Vec = (0..h).map(|y| g[y * w + x]).collect(); + let mut d = vec![0.0; h]; + let (mut v, mut z) = (vec![0usize; h], vec![0.0f64; h + 1]); + dt1(&f, &mut d, &mut v, &mut z); + d + }) + .collect(); + for (x, c) in cols.iter().enumerate() { + for y in 0..h { + g[y * w + x] = c[y]; + } + } + // Rows. + g.par_chunks_mut(w).for_each(|row| { + let f = row.to_vec(); + let (mut v, mut z) = (vec![0usize; w], vec![0.0f64; w + 1]); + dt1(&f, row, &mut v, &mut z); + }); + g.par_iter_mut().for_each(|v| *v = v.sqrt()); + g +} + +/// Signed distance to a binary matte's edge: positive inside (distance to the nearest +/// background pixel centre, minus ½), negative outside. +pub fn signed_distance(m: &[u8], w: usize, h: usize) -> Vec { + let fg: Vec = m.iter().map(|v| *v != 0).collect(); + let bg: Vec = m.iter().map(|v| *v == 0).collect(); + let din = distance(&bg, w, h); + let dout = distance(&fg, w, h); + m.iter().enumerate().map(|(i, v)| if *v != 0 { din[i] - 0.5 } else { 0.5 - dout[i] }).collect() +} + +/// Mean over the (2r+1)² window (clamped at the edges: averages only the pixels inside). +pub fn box_mean(src: &[f32], w: usize, h: usize, r: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + // Horizontal window sums per row. + let mut tmp = vec![0.0f64; w * h]; + tmp.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let s = &src[y * w..y * w + w]; + let mut acc = 0.0f64; + for x in 0..r.min(w) { + acc += s[x] as f64; + } + for (x, o) in row.iter_mut().enumerate() { + if x + r < w { + acc += s[x + r] as f64; + } + if x > r { + acc -= s[x - r - 1] as f64; + } + *o = acc; + } + }); + // Vertical prefix sums (row by row, cache friendly), then window differences. + let mut pre = vec![0.0f64; w * (h + 1)]; + for y in 0..h { + let (done, rest) = pre.split_at_mut((y + 1) * w); + let prev = &done[y * w..]; + let cur = &mut rest[..w]; + let t = &tmp[y * w..y * w + w]; + for x in 0..w { + cur[x] = prev[x] + t[x]; + } + } + let mut out = vec![0.0f32; w * h]; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let a = y.saturating_sub(r); + let b = (y + r + 1).min(h); + let cy = (b - a) as f64; + for (x, o) in row.iter_mut().enumerate() { + let cx = ((x + r + 1).min(w) - x.saturating_sub(r)) as f64; + *o = ((pre[b * w + x] - pre[a * w + x]) / (cx * cy)) as f32; + } + }); + out +} + +/// Approximate Gaussian blur (three box passes) with standard deviation `sigma` pixels. +pub fn gauss(src: &[f32], w: usize, h: usize, sigma: f64) -> Vec { + if sigma < 0.3 { + return src.to_vec(); + } + // Box radius whose three passes match the variance: 3·((2r+1)² − 1)/12 = σ². + let r = (((4.0 * sigma * sigma + 1.0).sqrt() - 1.0) / 2.0).round().max(1.0) as usize; + let a = box_mean(src, w, h, r); + let b = box_mean(&a, w, h, r); + box_mean(&b, w, h, r) +} + +fn inv3(m: [[f32; 3]; 3]) -> [[f32; 3]; 3] { + let d = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]); + let d = if d.abs() < 1e-20 { 1e-20 } else { d }; + [ + [(m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d, (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d, (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d], + [(m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d, (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d, (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d], + [(m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d, (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d, (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d], + ] +} + +/// The colour guided filter of `p` with guide `img` (RGB, w × h), window radius `r`, +/// regularisation `eps` (He et al. 2010, section 3.5). +pub fn guided_filter(img: &[[f32; 3]], p: &[f32], w: usize, h: usize, r: usize, eps: f32) -> Vec { + let n = w * h; + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let (ir, ig, ib) = (ch(0), ch(1), ch(2)); + let mul = |a: &[f32], b: &[f32]| -> Vec { a.iter().zip(b).map(|(x, y)| x * y).collect() }; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let (mr, mg, mb, mp) = (bm(&ir), bm(&ig), bm(&ib), bm(p)); + let (mrp, mgp, mbp) = (bm(&mul(&ir, p)), bm(&mul(&ig, p)), bm(&mul(&ib, p))); + let (vrr, vrg, vrb) = (bm(&mul(&ir, &ir)), bm(&mul(&ir, &ig)), bm(&mul(&ir, &ib))); + let (vgg, vgb, vbb) = (bm(&mul(&ig, &ig)), bm(&mul(&ig, &ib)), bm(&mul(&ib, &ib))); + let mut a = vec![[0.0f32; 3]; n]; + let mut b = vec![0.0f32; n]; + a.par_iter_mut().zip(b.par_iter_mut()).enumerate().for_each(|(i, (ai, bi))| { + let cov = [mrp[i] - mr[i] * mp[i], mgp[i] - mg[i] * mp[i], mbp[i] - mb[i] * mp[i]]; + let s = [ + [vrr[i] - mr[i] * mr[i] + eps, vrg[i] - mr[i] * mg[i], vrb[i] - mr[i] * mb[i]], + [vrg[i] - mr[i] * mg[i], vgg[i] - mg[i] * mg[i] + eps, vgb[i] - mg[i] * mb[i]], + [vrb[i] - mr[i] * mb[i], vgb[i] - mg[i] * mb[i], vbb[i] - mb[i] * mb[i] + eps], + ]; + let inv = inv3(s); + let av = [ + cov[0] * inv[0][0] + cov[1] * inv[1][0] + cov[2] * inv[2][0], + cov[0] * inv[0][1] + cov[1] * inv[1][1] + cov[2] * inv[2][1], + cov[0] * inv[0][2] + cov[1] * inv[1][2] + cov[2] * inv[2][2], + ]; + *ai = av; + *bi = mp[i] - av[0] * mr[i] - av[1] * mg[i] - av[2] * mb[i]; + }); + let (a0, a1, a2): (Vec, Vec, Vec) = (a.iter().map(|v| v[0]).collect(), a.iter().map(|v| v[1]).collect(), a.iter().map(|v| v[2]).collect()); + let (ma0, ma1, ma2, mbb) = (bm(&a0), bm(&a1), bm(&a2), bm(&b)); + (0..n).into_par_iter().map(|i| ma0[i] * ir[i] + ma1[i] * ig[i] + ma2[i] * ib[i] + mbb[i]).collect() +} + +/// Closed-form matting (A. Levin, D. Lischinski and Y. Weiss, "A Closed-Form Solution to Natural +/// Image Matting", IEEE PAMI 30(2), 2008): the alpha that minimises αᵀLα with the matting +/// Laplacian `L` of the colour image, with `known` pixels fixed to `init`. Solved by conjugate +/// gradients over the unknown pixels, starting from `init`; `L·x` is evaluated with box filters +/// (K. He, J. Sun and X. Tang, "Fast Matting Using Large Kernel Matting Laplacian Matrices", +/// CVPR 2010): `(L·x)ᵢ = Σ_{k∋i} (xᵢ − a_kᵀIᵢ − b_k)` with `(a_k, b_k)` the ridge-regression +/// fit of `x` to the colours in window k. +pub fn closed_form(img: &[[f32; 3]], init: &[f32], known: &[bool], w: usize, h: usize, r: usize, eps: f32, iters: usize) -> Vec { + let n = w * h; + if n == 0 || known.iter().all(|k| *k) { + return init.to_vec(); + } + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let ic = [ch(0), ch(1), ch(2)]; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let mu = [bm(&ic[0]), bm(&ic[1]), bm(&ic[2])]; + let prod = |a: usize, b: usize| -> Vec { ic[a].iter().zip(&ic[b]).map(|(x, y)| x * y).collect() }; + let mut cov = [[vec![], vec![], vec![]], [vec![], vec![], vec![]], [vec![], vec![], vec![]]]; + for a in 0..3 { + for b in a..3 { + let m = bm(&prod(a, b)); + cov[a][b] = m.clone(); + cov[b][a] = m; + } + } + let inv: Vec<[[f32; 3]; 3]> = (0..n) + .into_par_iter() + .map(|i| { + let mut s = [[0.0f32; 3]; 3]; + for a in 0..3 { + for b in 0..3 { + s[a][b] = cov[a][b][i] - mu[a][i] * mu[b][i] + if a == b { eps } else { 0.0 }; + } + } + inv3(s) + }) + .collect(); + // Pixels per window (= windows per pixel). + let cnt: Vec = { + let (rr, mut out) = (r as i64, vec![0.0f32; n]); + for (i, o) in out.iter_mut().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let cx = (x + rr).min(w as i64 - 1) - (x - rr).max(0) + 1; + let cy = (y + rr).min(h as i64 - 1) - (y - rr).max(0) + 1; + *o = (cx * cy) as f32; + } + out + }; + let lap = |x: &[f32]| -> Vec { + let pm = bm(x); + let ipm: Vec> = (0..3).map(|c| bm(&ic[c].iter().zip(x).map(|(a, b)| a * b).collect::>())).collect(); + let ab: Vec<[f32; 4]> = (0..n) + .into_par_iter() + .map(|i| { + let cv = [ipm[0][i] - mu[0][i] * pm[i], ipm[1][i] - mu[1][i] * pm[i], ipm[2][i] - mu[2][i] * pm[i]]; + let m = &inv[i]; + let a = [ + m[0][0] * cv[0] + m[0][1] * cv[1] + m[0][2] * cv[2], + m[1][0] * cv[0] + m[1][1] * cv[1] + m[1][2] * cv[2], + m[2][0] * cv[0] + m[2][1] * cv[1] + m[2][2] * cv[2], + ]; + [a[0], a[1], a[2], pm[i] - a[0] * mu[0][i] - a[1] * mu[1][i] - a[2] * mu[2][i]] + }) + .collect(); + let sa: Vec> = (0..3).map(|c| bm(&ab.iter().map(|v| v[c]).collect::>())).collect(); + let sb = bm(&ab.iter().map(|v| v[3]).collect::>()); + (0..n).into_par_iter().map(|i| cnt[i] * (x[i] - sa[0][i] * ic[0][i] - sa[1][i] * ic[1][i] - sa[2][i] * ic[2][i] - sb[i])).collect() + }; + // L_UU x = −(L α_K)_U. + let a0: Vec = (0..n).map(|i| if known[i] { init[i] } else { 0.0 }).collect(); + let la0 = lap(&a0); + let mut x: Vec = (0..n).map(|i| if known[i] { 0.0 } else { init[i] }).collect(); + let lx = lap(&x); + let mut res: Vec = (0..n).map(|i| if known[i] { 0.0 } else { (-la0[i] - lx[i]) as f64 }).collect(); + let mut p = res.clone(); + let mut rs: f64 = res.iter().map(|v| v * v).sum(); + let rs0 = rs.max(1e-30); + for _ in 0..iters { + if rs <= rs0 * 1e-8 { + break; + } + let pf: Vec = p.iter().map(|v| *v as f32).collect(); + let ap: Vec = lap(&pf).iter().enumerate().map(|(i, v)| if known[i] { 0.0 } else { *v as f64 }).collect(); + let pap: f64 = p.iter().zip(&ap).map(|(a, b)| a * b).sum(); + if pap <= 1e-30 { + break; + } + let alpha = rs / pap; + for i in 0..n { + x[i] += (alpha * p[i]) as f32; + res[i] -= alpha * ap[i]; + } + let rs_new: f64 = res.iter().map(|v| v * v).sum(); + let beta = rs_new / rs; + rs = rs_new; + for i in 0..n { + p[i] = res[i] + beta * p[i]; + } + } + (0..n).map(|i| if known[i] { init[i] } else { x[i].clamp(0.0, 1.0) }).collect() +} + +/// Contrast (0–100 %) around ½ on a soft alpha. +pub fn contrast(a: &mut [f32], pc: f64) { + if pc <= 0.0 { + return; + } + let k = 1.0 / (1.0 - 0.99 * (pc / 100.0).clamp(0.0, 1.0)) as f32; + a.par_iter_mut().for_each(|v| *v = (0.5 + (*v - 0.5) * k).clamp(0.0, 1.0)); +} + +/// Shift Edge (−100–100 %) on a soft alpha: positive values grow the matte (a gamma curve on α). +pub fn shift_soft(a: &mut [f32], pc: f64) { + if pc.abs() < 1e-9 { + return; + } + let g = 2f32.powf((-pc / 50.0) as f32); + a.par_iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0).powf(g)); +} + +/// Estimate the pure foreground colour of every pixel listed in `region` (alpha strictly +/// between 0 and 1) and mix it in by `amount` (0–1). `radius`: neighbourhood (pixels) that pure +/// background / foreground estimates are taken from. Returns the decontamination map (how much +/// each pixel's colour changed, 0–1). +pub fn decontaminate(img: &mut [[f32; 3]], alpha: &[f32], region: &[bool], w: usize, h: usize, radius: usize, amount: f32) -> Vec { + let n = w * h; + let mut map = vec![0.0f32; n]; + if !region.iter().any(|r| *r) || amount <= 0.0 { + return map; + } + let lo: Vec = alpha.iter().map(|a| if *a < 0.02 { 1.0 } else { 0.0 }).collect(); + let hi: Vec = alpha.iter().map(|a| if *a > 0.98 { 1.0 } else { 0.0 }).collect(); + // Windowed means of pure background / foreground colours, widening where none is near. + let est = |m: &[f32]| -> Vec> { + let mut out: Vec> = vec![None; n]; + let mut r = radius.max(2); + for _ in 0..4 { + let cnt = box_mean(m, w, h, r); + let sums: Vec> = (0..3).map(|c| box_mean(&img.iter().zip(m).map(|(p, k)| p[c] * k).collect::>(), w, h, r)).collect(); + let mut missing = false; + for i in 0..n { + if region[i] && out[i].is_none() { + if cnt[i] > 1e-4 { + out[i] = Some([sums[0][i] / cnt[i], sums[1][i] / cnt[i], sums[2][i] / cnt[i]]); + } else { + missing = true; + } + } + } + if !missing { + break; + } + r *= 2; + } + out + }; + let bgc = est(&lo); + let fgc = est(&hi); + for i in 0..n { + if !region[i] { + continue; + } + let a = alpha[i].clamp(0.0, 1.0); + if a <= 0.0 || a >= 1.0 { + continue; + } + let c = img[i]; + let f = match (bgc[i], fgc[i]) { + (Some(b), fbar) => { + let mut f = [0.0f32; 3]; + for k in 0..3 { + f[k] = ((c[k] - (1.0 - a) * b[k]) / a).clamp(0.0, 1.0); + } + // Low alphas amplify noise: lean on the nearby foreground colour there. + if let Some(fb) = fbar { + let t = ((0.25 - a) / 0.25).clamp(0.0, 1.0); + for k in 0..3 { + f[k] += (fb[k] - f[k]) * t; + } + } + f + } + (None, Some(fb)) => fb, + (None, None) => c, + }; + let mut d = 0.0f32; + for k in 0..3 { + let v = c[k] + (f[k] - c[k]) * amount; + d = d.max((v - c[k]).abs()); + img[i][k] = v; + } + map[i] = d.min(1.0); + } + map +} + +/// Sample a float plane bilinearly at continuous coordinates (pixel centres at +0.5), clamped. +pub fn sample(p: &[f32], w: usize, h: usize, x: f64, y: f64) -> f32 { + if w == 0 || h == 0 { + return 0.0; + } + let fx = (x - 0.5).clamp(0.0, (w - 1) as f64); + let fy = (y - 0.5).clamp(0.0, (h - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(w - 1), (y0 + 1).min(h - 1)); + let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); + let g = |x: usize, y: usize| p[y * w + x]; + let top = g(x0, y0) + (g(x1, y0) - g(x0, y0)) * tx; + let bot = g(x0, y1) + (g(x1, y1) - g(x0, y1)) * tx; + top + (bot - top) * ty +} + +/// Average of `a` sampled along the segment `[-v/2, v/2]` with `samples` taps (motion blur of a +/// matte moving by `v` pixels during the shutter). +pub fn motion_blur(a: &[f32], w: usize, h: usize, v: [f64; 2], samples: usize) -> Vec { + let n = samples.max(2); + if v[0].hypot(v[1]) < 0.25 { + return a.to_vec(); + } + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let mut s = 0.0; + for k in 0..n { + let t = k as f64 / (n - 1) as f64 - 0.5; + s += sample(a, w, h, x - v[0] * t, y - v[1] * t); + } + s / n as f32 + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn distance_matches_brute_force() { + let (w, h) = (23, 17); + let seed: Vec = (0..w * h).map(|i| (i * 7919) % 31 == 0).collect(); + let d = distance(&seed, w, h); + for y in 0..h { + for x in 0..w { + let mut b = f32::INFINITY; + for j in 0..h { + for i in 0..w { + if seed[j * w + i] { + b = b.min(((x as f32 - i as f32).powi(2) + (y as f32 - j as f32).powi(2)).sqrt()); + } + } + } + assert!((d[y * w + x] - b).abs() < 1e-3, "{x},{y}: {} vs {b}", d[y * w + x]); + } + } + } + + #[test] + fn guided_filter_recovers_a_linear_blend() { + // I = αF + (1 − α)B with constant F, B: the filtered binary matte is close to α. + let (w, h) = (64, 8); + let alpha: Vec = (0..w * h).map(|i| (((i % w) as f32 - 26.0) / 12.0).clamp(0.0, 1.0)).collect(); + let (f, b) = ([0.9f32, 0.2, 0.1], [0.1f32, 0.3, 0.8]); + let img: Vec<[f32; 3]> = alpha.iter().map(|a| [a * f[0] + (1.0 - a) * b[0], a * f[1] + (1.0 - a) * b[1], a * f[2] + (1.0 - a) * b[2]]).collect(); + let p: Vec = alpha.iter().map(|a| if *a >= 0.5 { 1.0 } else { 0.0 }).collect(); + let q = guided_filter(&img, &p, w, h, 8, 1e-5); + let err: f32 = q.iter().zip(&alpha).map(|(a, b)| (a.clamp(0.0, 1.0) - b).abs()).sum::() / (w * h) as f32; + assert!(err < 0.03, "mean error {err}"); + } +} diff --git a/crates/track/src/roto/maxflow.rs b/crates/track/src/roto/maxflow.rs new file mode 100644 index 000000000..dee4b010e --- /dev/null +++ b/crates/track/src/roto/maxflow.rs @@ -0,0 +1,438 @@ +//! Min-cut / max-flow on sparse graphs with source and sink terminals, by the +//! Boykov–Kolmogorov algorithm (Y. Boykov and V. Kolmogorov, "An Experimental Comparison of +//! Min-Cut/Max-Flow Algorithms for Energy Minimization in Vision", IEEE PAMI 26(9), 2004). +//! +//! Two search trees grow from the terminals (S from the source, T from the sink) through +//! non-saturated edges; when they touch, the path found is augmented; saturated tree edges turn +//! the nodes below them into *orphans*, which the adoption stage re-attaches to their tree (or +//! frees). Trees are reused between augmentations, which is what makes the method fast on the +//! grid graphs of image segmentation. The timestamp/distance heuristic of section 3.2.3 keeps +//! adoption paths short. +//! +//! Capacities are integers (callers scale their energies), so saturation tests are exact and +//! the result is deterministic. + +use std::collections::VecDeque; + +const NONE: u32 = u32::MAX; +const TERMINAL: u32 = u32::MAX - 1; +const ORPHAN: u32 = u32::MAX - 2; + +#[derive(Clone, Copy)] +struct Node { + first: u32, + /// Arc to the parent (from this node), or TERMINAL / ORPHAN / NONE (free). + parent: u32, + /// Residual terminal capacity: > 0 towards the source, < 0 towards the sink. + tr_cap: i64, + is_sink: bool, + active: bool, + ts: u32, + dist: u32, +} + +#[derive(Clone, Copy)] +struct Arc { + head: u32, + next: u32, + r_cap: i32, +} + +/// A graph for one min-cut computation. +pub struct Graph { + nodes: Vec, + arcs: Vec, + flow: i64, + time: u32, + queue: VecDeque, + orphans: VecDeque, +} + +/// Which side of the cut a node ended on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Segment { + Source, + Sink, +} + +impl Graph { + pub fn new(nodes: usize, edges_hint: usize) -> Graph { + Graph { + nodes: vec![Node { first: NONE, parent: NONE, tr_cap: 0, is_sink: false, active: false, ts: 0, dist: 0 }; nodes], + arcs: Vec::with_capacity(edges_hint * 2), + flow: 0, + time: 0, + queue: VecDeque::new(), + orphans: VecDeque::new(), + } + } + + pub fn len(&self) -> usize { + self.nodes.len() + } + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Terminal capacities of node `i`: `source` is paid when `i` ends on the sink side, `sink` + /// when it ends on the source side. + pub fn add_tweights(&mut self, i: usize, source: i64, sink: i64) { + let n = &mut self.nodes[i]; + let d = n.tr_cap; + let (s, t) = if d > 0 { (source + d, sink) } else { (source, sink - d) }; + self.flow += s.min(t); + n.tr_cap = s - t; + } + + /// An edge between `i` and `j` with capacity `cap` from i to j and `rev` from j to i. + pub fn add_edge(&mut self, i: usize, j: usize, cap: i32, rev: i32) { + debug_assert!(i != j); + let a = self.arcs.len() as u32; + self.arcs.push(Arc { head: j as u32, next: self.nodes[i].first, r_cap: cap }); + self.nodes[i].first = a; + self.arcs.push(Arc { head: i as u32, next: self.nodes[j].first, r_cap: rev }); + self.nodes[j].first = a + 1; + } + + #[inline] + fn set_active(&mut self, i: u32) { + let n = &mut self.nodes[i as usize]; + if !n.active { + n.active = true; + self.queue.push_back(i); + } + } + + /// Compute the maximum flow (= the minimum cut's cost). + pub fn maxflow(&mut self) -> i64 { + for i in 0..self.nodes.len() { + let n = &mut self.nodes[i]; + if n.tr_cap != 0 { + n.is_sink = n.tr_cap < 0; + n.parent = TERMINAL; + n.ts = 0; + n.dist = 1; + self.set_active(i as u32); + } + } + let mut current: Option = None; + loop { + let i = match current.take() { + Some(i) if self.nodes[i as usize].parent != NONE => i, + _ => loop { + let Some(i) = self.queue.pop_front() else { return self.flow }; + self.nodes[i as usize].active = false; + if self.nodes[i as usize].parent != NONE { + break i; + } + }, + }; + // Growth. + let mut found = NONE; + let ni = self.nodes[i as usize]; + let mut a = ni.first; + if !ni.is_sink { + while a != NONE { + let arc = self.arcs[a as usize]; + if arc.r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = false; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if nj.is_sink { + found = a; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } else { + while a != NONE { + let arc = self.arcs[a as usize]; + if self.arcs[(a ^ 1) as usize].r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = true; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if !nj.is_sink { + found = a ^ 1; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } + self.time = self.time.wrapping_add(1); + if found != NONE { + // Keep `i` current: it may have more paths. + current = Some(i); + self.augment(found); + self.adopt(); + } + } + } + + fn augment(&mut self, middle: u32) { + let mut b = self.arcs[middle as usize].r_cap as i64; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[(a ^ 1) as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(self.nodes[i as usize].tr_cap); + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[a as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(-self.nodes[i as usize].tr_cap); + let bi = b as i32; + self.arcs[(middle ^ 1) as usize].r_cap += bi; + self.arcs[middle as usize].r_cap -= bi; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[a as usize].r_cap += bi; + self.arcs[(a ^ 1) as usize].r_cap -= bi; + if self.arcs[(a ^ 1) as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap -= b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[(a ^ 1) as usize].r_cap += bi; + self.arcs[a as usize].r_cap -= bi; + if self.arcs[a as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap += b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + self.flow += b; + } + + fn make_orphan(&mut self, i: u32) { + self.nodes[i as usize].parent = ORPHAN; + self.orphans.push_back(i); + } + + /// Whether node `j`'s path to its terminal is intact; returns its distance (marking the + /// path with the current time) or None. + fn origin_dist(&mut self, j: u32) -> Option { + let mut d = 0u32; + let mut k = j; + loop { + let n = self.nodes[k as usize]; + if n.ts == self.time { + d += n.dist; + break; + } + let a = n.parent; + d += 1; + if a == TERMINAL { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = 1; + break; + } + if a == ORPHAN || a == NONE { + return None; + } + k = self.arcs[a as usize].head; + } + // Mark the path. + let total = d; + let mut k = j; + let mut dd = d; + while self.nodes[k as usize].ts != self.time { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = dd; + dd -= 1; + k = self.arcs[n.parent as usize].head; + } + Some(total) + } + + fn adopt(&mut self) { + while let Some(i) = self.orphans.pop_front() { + let sink = self.nodes[i as usize].is_sink; + let mut best = NONE; + let mut best_d = u32::MAX; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + // S orphan: the parent must push flow into i (residual on j→i = sister of i→j). + // T orphan: i must push flow into the parent (residual on i→j). + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink + && nj.parent != NONE + && let Some(d) = self.origin_dist(j) + && d < best_d + { + best = a; + best_d = d; + } + } + a = arc.next; + } + if best != NONE { + let n = &mut self.nodes[i as usize]; + n.parent = best; + n.ts = self.time; + n.dist = best_d + 1; + continue; + } + // No parent: free the node; its children become orphans, neighbours that could + // reach it become active. + self.nodes[i as usize].ts = 0; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink && nj.parent != NONE { + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + self.set_active(j); + } + let pa = nj.parent; + if pa != TERMINAL && pa != ORPHAN && self.arcs[pa as usize].head == i { + self.make_orphan(j); + } + } + a = arc.next; + } + self.nodes[i as usize].parent = NONE; + } + } + + /// The side of node `i` after [`Graph::maxflow`] (free nodes go to the sink). + pub fn segment(&self, i: usize) -> Segment { + let n = &self.nodes[i]; + if n.parent != NONE && !n.is_sink { Segment::Source } else { Segment::Sink } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Brute-force min cut over all labelings of a small graph. + fn brute(n: usize, t: &[(i64, i64)], e: &[(usize, usize, i32, i32)]) -> i64 { + let mut best = i64::MAX; + for m in 0..(1u32 << n) { + let src = |i: usize| m & (1 << i) != 0; + let mut c = 0i64; + for (i, (s, k)) in t.iter().enumerate() { + c += if src(i) { *k } else { *s }; + } + for (i, j, a, b) in e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + best = best.min(c); + } + best + } + + #[test] + fn matches_brute_force_on_random_graphs() { + let mut s = 12345u64; + let mut rnd = |m: u64| { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s % m + }; + for _ in 0..300 { + let n = 2 + rnd(9) as usize; + let t: Vec<(i64, i64)> = (0..n).map(|_| (rnd(20) as i64, rnd(20) as i64)).collect(); + let mut e = vec![]; + for _ in 0..rnd(20) { + let (i, j) = (rnd(n as u64) as usize, rnd(n as u64) as usize); + if i != j { + e.push((i, j, rnd(15) as i32, rnd(15) as i32)); + } + } + let mut g = Graph::new(n, e.len()); + for (i, (a, b)) in t.iter().enumerate() { + g.add_tweights(i, *a, *b); + } + for (i, j, a, b) in &e { + g.add_edge(*i, *j, *a, *b); + } + let f = g.maxflow(); + assert_eq!(f, brute(n, &t, &e)); + // The labeling realises the cut. + let src = |i: usize| g.segment(i) == Segment::Source; + let mut c = 0i64; + for (i, (a, b)) in t.iter().enumerate() { + c += if src(i) { *b } else { *a }; + } + for (i, j, a, b) in &e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + assert_eq!(c, f); + } + } +} diff --git a/crates/track/src/roto/mod.rs b/crates/track/src/roto/mod.rs new file mode 100644 index 000000000..393f743e0 --- /dev/null +++ b/crates/track/src/roto/mod.rs @@ -0,0 +1,754 @@ +//! Roto Brush segmentation and propagation, clean-room from published methods: +//! +//! - **Interactive graph-cut segmentation** (Y. Boykov and M.-P. Jolly, "Interactive Graph Cuts +//! for Optimal Boundary & Region Segmentation of Objects in N-D Images", ICCV 2001): the user's +//! foreground / background strokes are hard constraints, every other pixel gets a data cost +//! from colour models and neighbouring pixels a contrast-sensitive smoothness cost; the +//! minimum s–t cut ([`maxflow`], Boykov–Kolmogorov) is the segmentation. +//! - **Iterated colour models** (C. Rother, V. Kolmogorov and A. Blake, "GrabCut", SIGGRAPH +//! 2004): Gaussian mixtures ([`gmm`]) are fitted to the strokes, then re-fitted to the +//! segmentation and the cut repeated; the 8-connected n-link weight is +//! `γ/dist · exp(−β‖Iₚ − I_q‖²)` with `β = 1 / (2⟨‖Iₚ − I_q‖²⟩)`. +//! - **Coarse to fine**: large frames are segmented at a reduced size, then re-cut at full +//! resolution in a narrow band around the upsampled boundary (Lombaert et al., "A Multilevel +//! Banded Graph Cuts Method for Fast Image Segmentation", ICCV 2005). +//! - **Propagation** (in the spirit of X. Bai et al., "Video SnapCut", SIGGRAPH 2009): the +//! previous frame's matte is warped to the next frame by optical flow +//! (`effectcraft_raster::flow::block_flow`), and the frame re-segmented by graph cut only in a +//! band of **Search Radius** pixels around the warped boundary, with the previous frame's +//! colour models and the warped matte as a shape prior. Correction strokes on a frame re-cut +//! the whole frame with the warped matte as a soft prior, and propagation restarts from there. +//! +//! Mattes are in layer pixels at a given scale (1 = full resolution). Strokes are stored in +//! layer pixels ([`RotoData`], serde) and keyed per frame so that each frame's result is a +//! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes +//! on the way out to it ([`RotoData::chain_keys`]). + +pub mod gmm; +pub mod matting; +pub mod maxflow; +pub mod refine; +pub mod rle; + +use std::collections::BTreeMap; + +use effectcraft_raster::Image; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +pub use gmm::Gmm; +use maxflow::{Graph, Segment}; + +/// What a stroke paints. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum StrokeKind { + /// Roto Brush (green): foreground. + Fg, + /// Roto Brush with Alt (red): background. + Bg, + /// Refine Edge tool: mark a band for edge matting. + Refine, + /// Refine Edge tool with Alt: remove from the band. + RefineErase, +} + +impl StrokeKind { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "fg" | "foreground" => StrokeKind::Fg, + "bg" | "background" => StrokeKind::Bg, + "refine" | "refineedge" => StrokeKind::Refine, + "refineerase" | "refine-erase" | "erase" => StrokeKind::RefineErase, + _ => return None, + }) + } + pub fn is_segmentation(self) -> bool { + matches!(self, StrokeKind::Fg | StrokeKind::Bg) + } +} + +/// One stroke: a polyline swept by a disc of `radius` layer pixels, on layer frame `frame`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Stroke { + pub kind: StrokeKind, + pub frame: i64, + pub radius: f64, + pub points: Vec<[f64; 2]>, +} + +/// A Roto Brush instance's user data: strokes, the base frame and the segmentation span +/// (inclusive layer frames). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoData { + pub base: Option, + pub span: [i64; 2], + pub strokes: Vec, +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +impl RotoData { + pub fn from_json(s: &str) -> RotoData { + if s.trim().is_empty() { + return RotoData::default(); + } + serde_json::from_str(s).unwrap_or_default() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn is_empty(&self) -> bool { + self.base.is_none() || !self.strokes.iter().any(|s| s.kind.is_segmentation()) + } + pub fn strokes_at(&self, frame: i64) -> Vec<&Stroke> { + self.strokes.iter().filter(|s| s.frame == frame).collect() + } + /// Frames that carry strokes. + pub fn stroke_frames(&self) -> Vec { + let mut v: Vec = self.strokes.iter().map(|s| s.frame).collect(); + v.sort_unstable(); + v.dedup(); + v + } + pub fn in_span(&self, f: i64) -> bool { + self.base.is_some() && f >= self.span[0] && f <= self.span[1] + } + /// Add a stroke: the first segmentation stroke sets the base frame and a span of + /// `default_span` frames each side (clamped to `limits`); strokes outside the span extend it. + pub fn add(&mut self, s: Stroke, default_span: i64, limits: [i64; 2]) { + let f = s.frame; + match self.base { + None if s.kind.is_segmentation() => { + self.base = Some(f); + self.span = [(f - default_span).max(limits[0]), (f + default_span).min(limits[1]).max(f)]; + } + Some(_) => { + self.span[0] = self.span[0].min(f); + self.span[1] = self.span[1].max(f); + } + None => {} + } + self.strokes.push(s); + } + /// Largest Refine Edge stroke radius (layer pixels). + pub fn refine_radius(&self) -> f64 { + self.strokes.iter().filter(|s| matches!(s.kind, StrokeKind::Refine)).map(|s| s.radius).fold(0.0, f64::max) + } + /// Content key of every frame in the span: the base frame's from `seed` (what the frames + /// and settings are) and its strokes, every other frame's from its neighbour towards the + /// base and its own strokes. + pub fn chain_keys(&self, seed: u64) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(base) = self.base else { return out }; + let key = |prev: u64, f: i64| { + let mut h = prev ^ 0x9e37_79b9_7f4a_7c15; + fnv(&mut h, &f.to_le_bytes()); + for s in self.strokes.iter().filter(|s| s.frame == f) { + fnv(&mut h, serde_json::to_string(s).unwrap_or_default().as_bytes()); + } + h + }; + let k0 = key(seed, base); + out.insert(base, k0); + let mut k = k0; + for f in base + 1..=self.span[1] { + k = key(k, f); + out.insert(f, k); + } + k = k0; + for f in (self.span[0]..base).rev() { + k = key(k, f); + out.insert(f, k); + } + out + } + /// The neighbour a frame is propagated from (None for the base frame and outside the span). + pub fn source_of(&self, f: i64) -> Option { + let base = self.base?; + if !self.in_span(f) || f == base { + return None; + } + Some(if f > base { f - 1 } else { f + 1 }) + } +} + +/// Segmentation settings (the effect's Roto Brush parameters, in frame pixels). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SegOpts { + /// Pixels of the matte per layer pixel. + pub scale: f64, + /// Search Radius (layer pixels). + pub search_radius: f64, + /// Motion Threshold / Motion Damping, percent. + pub motion_threshold: f64, + pub motion_damping: f64, + /// Quality ▸ Best. + pub best: bool, +} + +impl Default for SegOpts { + fn default() -> Self { + SegOpts { scale: 1.0, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } + } +} + +/// One frame's segmentation: the binary matte, the Refine Edge band and the colour models. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FrameSeg { + pub w: usize, + pub h: usize, + pub matte: Vec, + pub refine: Vec, + pub fg: Gmm, + pub bg: Gmm, +} + +impl FrameSeg { + pub fn empty(w: usize, h: usize) -> FrameSeg { + FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], ..Default::default() } + } + pub fn area(&self) -> usize { + self.matte.iter().filter(|v| **v != 0).count() + } + /// Centroid of the matte (pixels), if any. + pub fn centroid(&self) -> Option<[f64; 2]> { + let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0); + for (i, v) in self.matte.iter().enumerate() { + if *v != 0 { + sx += (i % self.w) as f64 + 0.5; + sy += (i / self.w) as f64 + 0.5; + n += 1.0; + } + } + (n > 0.0).then(|| [sx / n, sy / n]) + } +} + +/// Straight RGB of an image (transparent pixels are black). +pub fn rgb_of(img: &Image) -> Vec<[f32; 3]> { + img.data.par_iter().map(|p| if p[3] > 1e-6 { [p[0] / p[3], p[1] / p[3], p[2] / p[3]] } else { [0.0; 3] }).collect() +} + +/// Pixels covered by `strokes` (of the given kinds) at `scale`. +pub fn rasterize(strokes: &[&Stroke], kinds: &[StrokeKind], scale: f64, w: usize, h: usize) -> Vec { + let mut m = vec![false; w * h]; + for s in strokes.iter().filter(|s| kinds.contains(&s.kind)) { + let r = (s.radius * scale).max(0.5); + let pts: Vec<[f64; 2]> = s.points.iter().map(|p| [p[0] * scale, p[1] * scale]).collect(); + let segs: Vec<([f64; 2], [f64; 2])> = if pts.len() == 1 { vec![(pts[0], pts[0])] } else { pts.windows(2).map(|w| (w[0], w[1])).collect() }; + for (a, b) in segs { + let x0 = (a[0].min(b[0]) - r).floor().max(0.0) as usize; + let x1 = ((a[0].max(b[0]) + r).ceil().max(0.0) as usize).min(w); + let y0 = (a[1].min(b[1]) - r).floor().max(0.0) as usize; + let y1 = ((a[1].max(b[1]) + r).ceil().max(0.0) as usize).min(h); + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + for y in y0..y1 { + for x in x0..x1 { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + let q = [a[0] + d[0] * t, a[1] + d[1] * t]; + if (p[0] - q[0]).hypot(p[1] - q[1]) <= r { + m[y * w + x] = true; + } + } + } + } + } + m +} + +/// Hard constraints: 1 = foreground, 2 = background, 0 = free. +const HARD_FG: u8 = 1; +const HARD_BG: u8 = 2; + +/// Graph-cut energy weights. +const GAMMA: f64 = 50.0; +const PRIOR: f64 = 1.0; +/// Fixed-point scale of the energies. +const K: f64 = 16.0; +const INF: i64 = 1 << 40; + +const NB: [(i64, i64, bool); 8] = [(1, 0, true), (0, 1, true), (1, 1, true), (-1, 1, true), (-1, 0, false), (0, -1, false), (-1, -1, false), (1, -1, false)]; + +fn beta(img: &[[f32; 3]], free: &[bool], w: usize, h: usize) -> f64 { + let (mut s, mut n) = (0.0f64, 0.0f64); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if !free[i] { + continue; + } + let c = img[i]; + for (dx, dy) in [(1usize, 0usize), (0, 1)] { + let (nx, ny) = (x + dx, y + dy); + if nx < w && ny < h { + let q = img[ny * w + nx]; + s += ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + n += 1.0; + } + } + } + } + if s <= 1e-12 { 1.0 } else { n / (2.0 * s) } +} + +/// One min-cut over the `free` pixels; others keep `labels`. `prior`: P(foreground) per pixel. +fn cut(img: &[[f32; 3]], w: usize, h: usize, labels: &mut [u8], free: &[bool], hard: &[u8], fg: &Gmm, bg: &Gmm, prior: Option<&[f32]>) { + let mut idx = vec![u32::MAX; w * h]; + let mut nodes = vec![]; + for i in 0..w * h { + if free[i] { + idx[i] = nodes.len() as u32; + nodes.push(i); + } + } + if nodes.is_empty() { + return; + } + let b = beta(img, free, w, h); + let costs: Vec<(f64, f64)> = nodes + .par_iter() + .map(|&i| { + let c = img[i]; + let (mut cf, mut cb) = (fg.cost(c), bg.cost(c)); + if let Some(p) = prior { + let pf = (p[i] as f64).clamp(0.02, 0.98); + cf += PRIOR * -pf.ln(); + cb += PRIOR * -(1.0 - pf).ln(); + } + (cf, cb) + }) + .collect(); + let mut g = Graph::new(nodes.len(), nodes.len() * 4); + for (n, &i) in nodes.iter().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let (cf, cb) = costs[n]; + let m = cf.min(cb); + let (mut src, mut snk) = (((cb - m) * K).round() as i64, ((cf - m) * K).round() as i64); + match hard[i] { + HARD_FG => (src, snk) = (INF, 0), + HARD_BG => (src, snk) = (0, INF), + _ => {} + } + g.add_tweights(n, src, snk); + let c = img[i]; + for (dx, dy, fwd) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + let q = img[j]; + let d2 = ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + let dist = if dx != 0 && dy != 0 { std::f64::consts::SQRT_2 } else { 1.0 }; + let wt = ((GAMMA / dist * (-b * d2).exp()) * K).round() as i32; + if wt <= 0 { + continue; + } + if idx[j] == u32::MAX { + // Fixed neighbour: disagreeing with it costs the edge. + if labels[j] != 0 { + g.add_tweights(n, wt as i64, 0); + } else { + g.add_tweights(n, 0, wt as i64); + } + } else if fwd { + g.add_edge(n, idx[j] as usize, wt, wt); + } + } + } + g.maxflow(); + for (n, &i) in nodes.iter().enumerate() { + labels[i] = (g.segment(n) == Segment::Source) as u8; + } +} + +/// Colour samples of the pixels where `sel` holds. +fn samples(img: &[[f32; 3]], sel: impl Fn(usize) -> bool) -> Vec<[f32; 3]> { + img.iter().enumerate().filter(|(i, _)| sel(*i)).map(|(_, c)| *c).collect() +} + +const COMPONENTS: usize = 5; +const MAX_SAMPLES: usize = 24_000; + +fn fit_models(img: &[[f32; 3]], labels: &[u8]) -> (Gmm, Gmm) { + let fg = samples(img, |i| labels[i] != 0); + let bg = samples(img, |i| labels[i] == 0); + (Gmm::fit(&fg, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bg, COMPONENTS, MAX_SAMPLES)) +} + +/// Keep the foreground components (8-connected) that touch `seeds`; fill small background +/// holes that touch neither the frame edge nor a background seed. +fn clean(labels: &mut [u8], seeds: &[bool], bg_seeds: &[bool], w: usize, h: usize) { + let n = w * h; + let mut comp = vec![u32::MAX; n]; + let mut stack = vec![]; + let mut fg_area = 0usize; + for start in 0..n { + if comp[start] != u32::MAX { + continue; + } + let lab = labels[start]; + let id = start as u32; + comp[start] = id; + stack.push(start); + let mut members = vec![]; + let (mut seeded, mut edge, mut bgs) = (false, false, false); + while let Some(i) = stack.pop() { + members.push(i); + seeded |= seeds[i]; + bgs |= bg_seeds[i]; + let (x, y) = ((i % w) as i64, (i / w) as i64); + edge |= x == 0 || y == 0 || x == w as i64 - 1 || y == h as i64 - 1; + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if comp[j] == u32::MAX && labels[j] == lab { + comp[j] = id; + stack.push(j); + } + } + } + if lab != 0 { + if seeded { + fg_area += members.len(); + } else { + for i in members { + labels[i] = 0; + } + } + } else if !edge && !bgs { + // Remember holes as negative candidates (resolved below once fg_area is known). + for i in members { + comp[i] = u32::MAX - 1; + } + } + } + // Fill holes smaller than 2 % of the foreground. + let mut seen = vec![false; n]; + for start in 0..n { + if comp[start] != u32::MAX - 1 || seen[start] || labels[start] != 0 { + continue; + } + let mut members = vec![start]; + seen[start] = true; + let mut k = 0; + while k < members.len() { + let i = members[k]; + k += 1; + let (x, y) = ((i % w) as i64, (i / w) as i64); + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if !seen[j] && comp[j] == u32::MAX - 1 && labels[j] == 0 { + seen[j] = true; + members.push(j); + } + } + } + // A hole is enclosed by foreground that survived the component filter. + let enclosed = members.iter().all(|&i| { + let (x, y) = ((i % w) as i64, (i / w) as i64); + NB.iter().all(|(dx, dy, _)| { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + return false; + } + let j = ny as usize * w + nx as usize; + labels[j] != 0 || comp[j] == u32::MAX - 1 + }) + }); + if enclosed && (members.len() as f64) < 0.02 * fg_area as f64 { + for i in members { + labels[i] = 1; + } + } + } +} + +fn downsample_rgb(img: &[[f32; 3]], w: usize, h: usize, f: usize) -> (Vec<[f32; 3]>, usize, usize) { + let (cw, ch) = (w.div_ceil(f), h.div_ceil(f)); + let out = (0..cw * ch) + .into_par_iter() + .map(|i| { + let (cx, cy) = (i % cw, i / cw); + let mut s = [0.0f32; 3]; + let mut n = 0.0; + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let c = img[y * w + x]; + s[0] += c[0]; + s[1] += c[1]; + s[2] += c[2]; + n += 1.0; + } + } + [s[0] / n, s[1] / n, s[2] / n] + }) + .collect(); + (out, cw, ch) +} + +/// What a frame is segmented with besides its strokes: the matte propagated from its neighbour. +pub struct Prior { + /// Warped neighbour matte, soft (0–1). + pub soft: Vec, + /// The neighbour's colour models. + pub fg: Gmm, + pub bg: Gmm, + /// Warped Refine Edge band. + pub refine: Vec, +} + +/// Segment a frame from its strokes (and a propagated prior): GrabCut iterations on a reduced +/// copy, then a full-resolution cut in a band around the boundary. +pub fn segment(img: &Image, strokes: &[&Stroke], prior: Option<&Prior>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (img.width as usize, img.height as usize); + if w == 0 || h == 0 { + return FrameSeg::empty(w, h); + } + let rgb = rgb_of(img); + let fgs = rasterize(strokes, &[StrokeKind::Fg], opts.scale, w, h); + let bgs = rasterize(strokes, &[StrokeKind::Bg], opts.scale, w, h); + let hard: Vec = fgs + .iter() + .zip(&bgs) + .map(|(f, b)| { + if *b { + HARD_BG + } else if *f { + HARD_FG + } else { + 0 + } + }) + .collect(); + let soft_prior: Option> = prior.map(|p| matting::gauss(&p.soft, w, h, (opts.search_radius * opts.scale / 4.0).max(1.0))); + // Initial colour models: the strokes (and the propagated matte). + let (fg0, bg0) = match prior { + Some(p) => { + let fgc = samples(&rgb, |i| (hard[i] == HARD_FG) || (hard[i] == 0 && p.soft[i] >= 0.5)); + let bgc = samples(&rgb, |i| (hard[i] == HARD_BG) || (hard[i] == 0 && p.soft[i] < 0.5)); + let f = Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES); + let b = Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES); + (if f.is_empty() { p.fg.clone() } else { f }, if b.is_empty() { p.bg.clone() } else { b }) + } + None => { + let fgc = samples(&rgb, |i| hard[i] == HARD_FG); + let mut bgc = samples(&rgb, |i| hard[i] == HARD_BG); + if bgc.is_empty() { + // No background strokes: what is far from the foreground strokes. + let d = matting::distance(&fgs, w, h); + let r = strokes.iter().filter(|s| s.kind == StrokeKind::Fg).map(|s| s.radius).fold(1.0, f64::max) * opts.scale; + let far = (4.0 * r).max(0.25 * w.max(h) as f64) as f32; + bgc = samples(&rgb, |i| d[i] > far); + if bgc.is_empty() { + bgc = samples(&rgb, |i| hard[i] != HARD_FG); + } + } + (Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES)) + } + }; + let iters = if opts.best { 4 } else { 3 }; + let work = if opts.best { 960 } else { 480 }; + let f = w.max(h).div_ceil(work).max(1); + let mut labels: Vec = match &soft_prior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => fgs.iter().map(|v| *v as u8).collect(), + }; + for (l, hd) in labels.iter_mut().zip(&hard) { + if *hd != 0 { + *l = (*hd == HARD_FG) as u8; + } + } + let (mut fgm, mut bgm) = (fg0, bg0); + if f > 1 { + let (crgb, cw, ch) = downsample_rgb(&rgb, w, h, f); + let mut chard = vec![0u8; cw * ch]; + let mut cprior = soft_prior.as_ref().map(|_| vec![0.0f32; cw * ch]); + for cy in 0..ch { + for cx in 0..cw { + let (mut anyf, mut anyb, mut s, mut n) = (false, false, 0.0f32, 0.0f32); + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let i = y * w + x; + anyf |= hard[i] == HARD_FG; + anyb |= hard[i] == HARD_BG; + if let Some(p) = &soft_prior { + s += p[i]; + } + n += 1.0; + } + } + chard[cy * cw + cx] = match (anyf, anyb) { + (true, false) => HARD_FG, + (false, true) => HARD_BG, + _ => 0, + }; + if let Some(cp) = &mut cprior { + cp[cy * cw + cx] = s / n; + } + } + } + let mut cl: Vec = match &cprior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => chard.iter().map(|v| (*v == HARD_FG) as u8).collect(), + }; + let free = vec![true; cw * ch]; + for it in 0..iters { + cut(&crgb, cw, ch, &mut cl, &free, &chard, &fgm, &bgm, cprior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&crgb, &cl); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + for y in 0..h { + for x in 0..w { + let i = y * w + x; + labels[i] = if hard[i] != 0 { (hard[i] == HARD_FG) as u8 } else { cl[(y / f) * cw + x / f] }; + } + } + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + // Full-resolution band. + let sd = matting::signed_distance(&labels, w, h); + let band = f as f32 + 1.5; + let free: Vec = sd.iter().map(|d| d.abs() <= band).collect(); + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + } else { + let free = vec![true; w * h]; + for it in 0..iters { + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + } + // Keep what the strokes (or the prior's core) select. + let mut seeds = fgs.clone(); + if let Some(p) = &soft_prior { + let core = core_of(p, w, h, opts.search_radius * opts.scale); + for (s, c) in seeds.iter_mut().zip(core) { + *s |= c; + } + } + clean(&mut labels, &seeds, &bgs, w, h); + finish(img, &rgb, labels, strokes, prior.map(|p| p.refine.as_slice()), opts, refine_radius) +} + +/// Pixels deep inside a soft matte (more than `r` from its edge), else the matte itself. +fn core_of(soft: &[f32], w: usize, h: usize, r: f64) -> Vec { + let bin: Vec = soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&bin, w, h); + let core: Vec = sd.iter().map(|d| *d as f64 > r).collect(); + if core.iter().any(|c| *c) { core } else { bin.iter().map(|v| *v != 0).collect() } +} + +/// Final models and Refine Edge band of a segmented frame. +fn finish(_img: &Image, rgb: &[[f32; 3]], labels: Vec, strokes: &[&Stroke], warped_refine: Option<&[u8]>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (_img.width as usize, _img.height as usize); + let (fg, bg) = fit_models(rgb, &labels); + let add = rasterize(strokes, &[StrokeKind::Refine], opts.scale, w, h); + let erase = rasterize(strokes, &[StrokeKind::RefineErase], opts.scale, w, h); + let near: Option> = warped_refine.filter(|r| r.iter().any(|v| *v != 0)).map(|_| matting::signed_distance(&labels, w, h)); + let rr = (refine_radius * opts.scale + 2.0) as f32; + let refine: Vec = (0..w * h) + .map(|i| { + let carried = match (warped_refine, &near) { + (Some(r), Some(sd)) => r[i] != 0 && sd[i].abs() <= rr, + _ => false, + }; + ((carried || add[i]) && !erase[i]) as u8 + }) + .collect(); + FrameSeg { w, h, matte: labels, refine, fg, bg } +} + +/// Warp the previous frame's segmentation onto the next frame by optical flow (next → prev), +/// with Motion Threshold / Damping applied to the vectors. +pub fn warp_prior(prev_img: &Image, prev: &FrameSeg, next_img: &Image, opts: &SegOpts) -> Prior { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let flow = effectcraft_raster::flow::block_flow(next_img, prev_img, 8, 4); + let thr = (opts.motion_threshold / 100.0 * opts.search_radius * opts.scale).max(0.0); + let damp = (1.0 - opts.motion_damping / 100.0).clamp(0.0, 1.0); + let src: Vec = prev.matte.iter().map(|v| *v as f32).collect(); + let pw = prev.w.min(w); + let ph = prev.h.min(h); + let res: Vec<(f32, u8)> = (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let v = flow.at(x, y); + let (mut vx, mut vy) = (v[0] as f64, v[1] as f64); + if vx.hypot(vy) < thr { + vx *= damp; + vy *= damp; + } + let (sx, sy) = (x + vx, y + vy); + let soft = if pw == prev.w && ph == prev.h { matting::sample(&src, prev.w, prev.h, sx, sy) } else { 0.0 }; + let (ix, iy) = (sx.floor().clamp(0.0, (prev.w.max(1) - 1) as f64) as usize, sy.floor().clamp(0.0, (prev.h.max(1) - 1) as f64) as usize); + let r = prev.refine.get(iy * prev.w + ix).copied().unwrap_or(0); + (soft, r) + }) + .collect(); + Prior { soft: res.iter().map(|r| r.0).collect(), fg: prev.fg.clone(), bg: prev.bg.clone(), refine: res.iter().map(|r| r.1).collect() } +} + +/// Propagate the previous frame's segmentation to the next frame: warp it, then re-cut in a +/// band of Search Radius pixels around the warped boundary. Strokes on the next frame re-cut +/// the whole frame with the warped matte as a soft prior instead. +pub fn propagate(prev_img: &Image, prev: &FrameSeg, next_img: &Image, strokes: &[&Stroke], opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let prior = warp_prior(prev_img, prev, next_img, opts); + if strokes.iter().any(|s| s.kind.is_segmentation()) { + return segment(next_img, strokes, Some(&prior), opts, refine_radius); + } + let rgb = rgb_of(next_img); + let mut labels: Vec = prior.soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&labels, w, h); + let r = (opts.search_radius * opts.scale).max(2.0) as f32; + let free: Vec = sd.iter().map(|d| d.abs() <= r).collect(); + let hard = vec![0u8; w * h]; + let soft = matting::gauss(&prior.soft, w, h, (r as f64 / 4.0).max(1.0)); + let (fg, bg) = if prev.fg.is_empty() || prev.bg.is_empty() { fit_models(&rgb, &labels) } else { (prev.fg.clone(), prev.bg.clone()) }; + cut(&rgb, w, h, &mut labels, &free, &hard, &fg, &bg, Some(&soft)); + let seeds = core_of(&prior.soft, w, h, r as f64); + clean(&mut labels, &seeds, &vec![false; w * h], w, h); + finish(next_img, &rgb, labels, strokes, Some(&prior.refine), opts, refine_radius) +} + +/// Intersection over union of two binary mattes. +pub fn iou(a: &[u8], b: &[u8]) -> f64 { + let (mut i, mut u) = (0usize, 0usize); + for (x, y) in a.iter().zip(b) { + let (x, y) = (*x != 0, *y != 0); + i += (x && y) as usize; + u += (x || y) as usize; + } + if u == 0 { 1.0 } else { i as f64 / u as f64 } +} + +#[cfg(test)] +mod tests; diff --git a/crates/track/src/roto/refine.rs b/crates/track/src/roto/refine.rs new file mode 100644 index 000000000..2a498e286 --- /dev/null +++ b/crates/track/src/roto/refine.rs @@ -0,0 +1,198 @@ +//! From a frame's binary segmentation to the final soft matte: the Roto Brush Matte and Refine +//! Edge Matte adjustments, edge matting inside the Refine Edge band (colour guided filter), +//! Reduce Chatter (temporal averaging with the neighbouring frames), matte motion blur and edge +//! colour decontamination. + +use super::FrameSeg; +use super::matting::{self, contrast, gauss, guided_filter, shift_soft, signed_distance}; + +/// The effect's matte parameters (After Effects' units: pixels and percent). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MatteParams { + // Roto Brush Matte. + pub rb_feather: f64, + pub rb_contrast: f64, + pub rb_shift: f64, + pub rb_chatter: f64, + // Refine Edge Matte. + pub smooth: f64, + pub feather: f64, + pub contrast: f64, + pub shift: f64, + pub chatter: f64, + pub motion_blur: bool, + pub mb_samples: usize, + pub shutter_angle: f64, + /// Higher Quality: closed-form matting after the guided-filter estimate. + pub higher_quality: bool, + pub decontaminate: bool, + pub decontamination: f64, + pub extend_where_smoothed: bool, + pub increase_radius: f64, +} + +impl Default for MatteParams { + fn default() -> Self { + MatteParams { + rb_feather: 0.0, + rb_contrast: 0.0, + rb_shift: 0.0, + rb_chatter: 0.0, + smooth: 1.0, + feather: 0.0, + contrast: 0.0, + shift: 0.0, + chatter: 0.0, + motion_blur: false, + mb_samples: 11, + shutter_angle: 180.0, + higher_quality: true, + decontaminate: true, + decontamination: 100.0, + extend_where_smoothed: true, + increase_radius: 0.0, + } + } +} + +fn shifted(seg: &FrameSeg, px: f64) -> Vec { + if px.abs() < 0.25 { + return seg.matte.iter().map(|v| *v as f32).collect(); + } + let sd = signed_distance(&seg.matte, seg.w, seg.h); + sd.iter().map(|d| ((*d as f64) > -px) as u8 as f32).collect() +} + +/// The soft matte of `seg` (all planes `seg.w × seg.h`), with `rgb` the frame's straight colour. +/// `prev` / `next`: the neighbouring frames' segmentations (Reduce Chatter, motion blur). +/// `band_radius`: the Refine Edge brush radius in these pixels; `scale`: pixels per layer pixel. +pub fn matte_alpha( + rgb: &[[f32; 3]], + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + p: &MatteParams, + band_radius: f64, + scale: f64, +) -> Vec { + let (w, h) = (seg.w, seg.h); + let n = w * h; + let same = |s: &&FrameSeg| s.w == w && s.h == h; + let (prev, next) = (prev.filter(same), next.filter(same)); + // Roto Brush Matte: Shift Edge (100 % = 10 px), Reduce Chatter, Feather, Contrast. + let shift_px = p.rb_shift / 100.0 * 10.0 * scale; + let mut a = shifted(seg, shift_px); + let nb: Vec> = [prev, next].into_iter().flatten().map(|s| shifted(s, shift_px)).collect(); + if p.rb_chatter > 0.0 && !nb.is_empty() { + let k = (p.rb_chatter / 100.0).clamp(0.0, 1.0) as f32 * (nb.len() as f32 / (nb.len() as f32 + 1.0)); + for i in 0..n { + let m = nb.iter().map(|v| v[i]).sum::() / nb.len() as f32; + a[i] += (m - a[i]) * k; + } + } + if p.rb_feather > 0.0 { + a = gauss(&a, w, h, p.rb_feather * scale / 2.0); + } + contrast(&mut a, p.rb_contrast); + // Refine Edge band: guided-filter matting. + if seg.refine.iter().any(|v| *v != 0) && seg.refine.len() == n { + let r = (band_radius * 1.0).round().max(2.0) as usize; + let eps = (2e-4 * (1.0 + p.smooth.max(0.0))) as f32; + // Work on the band's bounding box (plus the filter's reach). + let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0, 0); + for (i, v) in seg.refine.iter().enumerate() { + if *v != 0 { + let (x, y) = (i % w, i / w); + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x + 1); + y1 = y1.max(y + 1); + } + } + let pad = 2 * r + 2; + let (x0, y0, x1, y1) = (x0.saturating_sub(pad), y0.saturating_sub(pad), (x1 + pad).min(w), (y1 + pad).min(h)); + let (cw, ch) = (x1 - x0, y1 - y0); + let crop = |src: &dyn Fn(usize) -> f32| -> Vec { (0..cw * ch).map(|j| src((y0 + j / cw) * w + x0 + j % cw)).collect() }; + let cimg: Vec<[f32; 3]> = (0..cw * ch).map(|j| rgb[(y0 + j / cw) * w + x0 + j % cw]).collect(); + let cp = crop(&|i| a[i]); + let band: Vec = crop(&|i| seg.refine[i] as f32).iter().map(|v| *v > 0.0).collect(); + let outside: Vec = band.iter().map(|b| !b).collect(); + // Guided filter estimate, then closed-form matting with everything outside the band known. + let mut q = guided_filter(&cimg, &cp, cw, ch, r, eps); + for j in 0..cw * ch { + q[j] = if band[j] { q[j].clamp(0.0, 1.0) } else { cp[j] }; + } + if p.higher_quality { + q = matting::closed_form(&cimg, &q, &outside, cw, ch, 1, 1e-5 * (1.0 + p.smooth.max(0.0)) as f32, 50); + } + // Refine Edge Matte adjustments. + if p.smooth > 1.0 { + q = gauss(&q, cw, ch, (p.smooth - 1.0) * 0.5 * scale); + } + if p.feather > 0.0 { + q = gauss(&q, cw, ch, p.feather * scale / 2.0); + } + contrast(&mut q, p.contrast); + shift_soft(&mut q, p.shift); + // Blend in, fading over the band's last two pixels. + let din = matting::distance(&outside, cw, ch); + for j in 0..cw * ch { + if !band[j] { + continue; + } + let wgt = (din[j] / 2.0).clamp(0.0, 1.0); + let i = (y0 + j / cw) * w + x0 + j % cw; + a[i] += (q[j] - a[i]) * wgt; + } + // Refine-stage Reduce Chatter: towards the neighbours' mattes inside the band. + if p.chatter > 0.0 && !nb.is_empty() { + let k = (p.chatter / 100.0).clamp(0.0, 1.0) as f32 * 0.5; + let soft: Vec> = nb.iter().map(|v| gauss(v, w, h, (r as f64 / 2.0).max(1.0))).collect(); + for i in 0..n { + if seg.refine[i] != 0 { + let m = soft.iter().map(|v| v[i]).sum::() / soft.len() as f32; + a[i] += (m - a[i]) * k; + } + } + } + } + // Motion blur: the matte's motion between the neighbouring frames. + if p.motion_blur { + let c = seg.centroid(); + let v = match (prev.and_then(FrameSeg::centroid), next.and_then(FrameSeg::centroid), c) { + (Some(a), Some(b), _) => Some([(b[0] - a[0]) / 2.0, (b[1] - a[1]) / 2.0]), + (Some(a), None, Some(c)) => Some([c[0] - a[0], c[1] - a[1]]), + (None, Some(b), Some(c)) => Some([b[0] - c[0], b[1] - c[1]]), + _ => None, + }; + if let Some(v) = v { + let s = p.shutter_angle / 360.0; + a = matting::motion_blur(&a, w, h, [v[0] * s, v[1] * s], p.mb_samples); + } + } + a.iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0)); + a +} + +/// Decontaminate edge colours of `rgb` in place for the matte `alpha` (see +/// [`matting::decontaminate`]); returns the decontamination map. `band`: the Refine Edge band +/// (pixels where decontamination always applies). +pub fn decontaminate(rgb: &mut [[f32; 3]], alpha: &[f32], band: &[u8], w: usize, h: usize, p: &MatteParams, band_radius: f64, scale: f64) -> Vec { + if !p.decontaminate { + return vec![0.0; w * h]; + } + let grow = p.increase_radius * scale; + let in_band: Vec = if band.len() == w * h && band.iter().any(|v| *v != 0) { + if grow > 0.5 { + let d = matting::distance(&band.iter().map(|v| *v != 0).collect::>(), w, h); + d.iter().map(|v| (*v as f64) <= grow).collect() + } else { + band.iter().map(|v| *v != 0).collect() + } + } else { + vec![false; w * h] + }; + let region: Vec = (0..w * h).map(|i| alpha[i] > 0.004 && alpha[i] < 0.996 && (in_band[i] || p.extend_where_smoothed)).collect(); + let radius = (band_radius + grow).max(4.0 * scale).round() as usize; + matting::decontaminate(rgb, alpha, ®ion, w, h, radius, (p.decontamination / 100.0).clamp(0.0, 1.0) as f32) +} diff --git a/crates/track/src/roto/rle.rs b/crates/track/src/roto/rle.rs new file mode 100644 index 000000000..308b2689a --- /dev/null +++ b/crates/track/src/roto/rle.rs @@ -0,0 +1,136 @@ +//! Compact text encodings of mattes for project storage: run-length coding (LEB128 varints) +//! wrapped in standard base64 (RFC 4648 §4). + +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +pub fn base64_encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let b = [c[0], *c.get(1).unwrap_or(&0), *c.get(2).unwrap_or(&0)]; + let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32; + out.push(B64[(n >> 18) as usize & 63] as char); + out.push(B64[(n >> 12) as usize & 63] as char); + out.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' }); + } + out +} + +pub fn base64_decode(s: &str) -> Option> { + let val = |c: u8| -> Option { + Some(match c { + b'A'..=b'Z' => (c - b'A') as u32, + b'a'..=b'z' => (c - b'a' + 26) as u32, + b'0'..=b'9' => (c - b'0' + 52) as u32, + b'+' => 62, + b'/' => 63, + _ => return None, + }) + }; + let bytes: Vec = s.bytes().filter(|c| !c.is_ascii_whitespace()).collect(); + if !bytes.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(bytes.len() / 4 * 3); + for c in bytes.chunks(4) { + let pad = c.iter().rev().take_while(|b| **b == b'=').count(); + let mut n = 0u32; + for (i, b) in c.iter().enumerate() { + n |= if *b == b'=' { 0 } else { val(*b)? } << (18 - 6 * i); + } + out.push((n >> 16) as u8); + if pad < 2 { + out.push((n >> 8) as u8); + } + if pad < 1 { + out.push(n as u8); + } + } + Some(out) +} + +fn put(out: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +fn get(data: &[u8], pos: &mut usize) -> Option { + let mut v = 0u64; + let mut shift = 0; + loop { + let b = *data.get(*pos)?; + *pos += 1; + v |= ((b & 0x7f) as u64) << shift; + if b & 0x80 == 0 { + return Some(v); + } + shift += 7; + if shift > 63 { + return None; + } + } +} + +/// Encode a byte plane as (value, run length) pairs. +pub fn encode(values: &[u8]) -> String { + let mut out = vec![]; + put(&mut out, values.len() as u64); + let mut i = 0; + while i < values.len() { + let v = values[i]; + let mut j = i + 1; + while j < values.len() && values[j] == v { + j += 1; + } + out.push(v); + put(&mut out, (j - i) as u64); + i = j; + } + base64_encode(&out) +} + +/// Decode [`encode`]'s output (None when malformed). +pub fn decode(s: &str) -> Option> { + let data = base64_decode(s)?; + let mut pos = 0; + let n = get(&data, &mut pos)? as usize; + if n > 1 << 30 { + return None; + } + let mut out = Vec::with_capacity(n); + while out.len() < n { + let v = *data.get(pos)?; + pos += 1; + let run = get(&data, &mut pos)? as usize; + if out.len() + run > n { + return None; + } + out.resize(out.len() + run, v); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips() { + for n in [0usize, 1, 2, 3, 4, 5, 100, 1000] { + let v: Vec = (0..n).map(|i| ((i / 7) % 3) as u8 * 100).collect(); + assert_eq!(decode(&encode(&v)).unwrap(), v); + let raw: Vec = (0..n).map(|i| (i * 31) as u8).collect(); + assert_eq!(base64_decode(&base64_encode(&raw)).unwrap(), raw); + } + assert_eq!(base64_encode(b"Man"), "TWFu"); + assert_eq!(base64_encode(b"Ma"), "TWE="); + assert!(decode("!!!!").is_none()); + } +} diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rs new file mode 100644 index 000000000..bec3f3783 --- /dev/null +++ b/crates/track/src/roto/tests.rs @@ -0,0 +1,331 @@ +//! Roto Brush on synthetic sequences: a textured disk moving over a textured background. + +use super::refine::{MatteParams, decontaminate, matte_alpha}; +use super::*; + +pub(crate) const W: u32 = 200; +pub(crate) const H: u32 = 150; +pub(crate) const R: f64 = 32.0; + +/// Smooth value noise in 0..1. +pub(crate) fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +pub(crate) fn fg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 11); + [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32] +} + +pub(crate) fn bg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 5); + let m = noise(x * 1.7, y * 1.7, 9); + [(0.05 + 0.25 * m) as f32, (0.3 + 0.4 * n) as f32, (0.45 + 0.4 * m) as f32] +} + +pub(crate) fn center(f: i64) -> [f64; 2] { + let t = f as f64; + [60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()] +} + +/// Frame `f`: the disk (texture moving with it) over the static background, with a 1-pixel +/// anti-aliased edge; `c` overrides the centre. +pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0u8; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let f = fg_color(p[0] - c[0], p[1] - c[1]); + let b = bg_color(p[0], p[1]); + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = (d <= R) as u8; + } + } + (img, gt) +} + +fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke { + Stroke { kind, frame, radius, points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect() } +} + +/// The strokes an artist would paint on frame `f`: one through the disk, one around it. +pub(crate) fn base_strokes(f: i64) -> Vec { + let c = center(f); + vec![ + line(StrokeKind::Fg, f, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0), + line(StrokeKind::Bg, f, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0), + line(StrokeKind::Bg, f, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0), + ] +} + +#[test] +fn strokes_segment_the_base_frame() { + let (img, gt) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); + // Deterministic. + let again = segment(&img, &refs, None, &SegOpts::default(), 0.0); + assert_eq!(seg, again); +} + +#[test] +fn coarse_to_fine_segments_large_frames() { + // A 2× upscaled frame goes through the reduced-size path (Standard works at 480 px). + let (small, _) = frame_at(center(0)); + let img = effectcraft_raster::resample(&small, W * 3, H * 3); + let c = center(0); + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut s| { + s.points.iter_mut().for_each(|p| *p = [p[0] * 3.0, p[1] * 3.0]); + s.radius *= 3.0; + s + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let gt: Vec = + (0..W * H * 9).map(|i| ((((i % (W * 3)) as f64 + 0.5) / 3.0 - c[0]).hypot((((i / (W * 3)) as f64 + 0.5) / 3.0) - c[1]) <= R) as u8).collect(); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); +} + +#[test] +fn propagation_follows_the_disk_for_20_frames() { + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts::default(); + let mut seg = segment(&img0, &refs, None, &opts, 0.0); + let mut prev = img0; + let mut worst: f64 = 1.0; + for f in 1..=20 { + let (img, gt) = frame_at(center(f)); + seg = propagate(&prev, &seg, &img, &[], &opts, 0.0); + let j = iou(&seg.matte, >); + worst = worst.min(j); + prev = img; + } + assert!(worst > 0.9, "worst IoU {worst}"); +} + +#[test] +fn a_correction_stroke_fixes_a_bad_frame() { + let opts = SegOpts::default(); + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg0 = segment(&img0, &refs, None, &opts, 0.0); + // The disk jumps far away on frame 1: propagation cannot follow. + let jump = [150.0, 90.0]; + let (img1, gt1) = frame_at(jump); + let bad = propagate(&img0, &seg0, &img1, &[], &opts, 0.0); + assert!(iou(&bad.matte, >1) < 0.5, "the jump should break propagation"); + // Paint the disk in and the old place out. + let c0 = center(0); + let fix = [ + line(StrokeKind::Fg, 1, [jump[0] - 15.0, jump[1]], [jump[0] + 15.0, jump[1]], 5.0), + line(StrokeKind::Bg, 1, [c0[0] - 10.0, c0[1] - 10.0], [c0[0] + 10.0, c0[1] + 10.0], 6.0), + ]; + let fr: Vec<&Stroke> = fix.iter().collect(); + let good = propagate(&img0, &seg0, &img1, &fr, &opts, 0.0); + let j = iou(&good.matte, >1); + assert!(j > 0.95, "corrected IoU {j}"); + // And propagation continues from the corrected frame. + let c2 = [jump[0] + 3.0, jump[1] + 1.0]; + let (img2, gt2) = frame_at(c2); + let next = propagate(&img1, &good, &img2, &[], &opts, 0.0); + assert!(iou(&next.matte, >2) > 0.93); +} + +/// A soft-edged disk (alpha ramp over `width` pixels) over a background. +fn soft_frame(c: [f64; 2], width: f64, fg: [f32; 3], bg: [f32; 3]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0.0f32; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = ((R + width / 2.0 - d) / width).clamp(0.0, 1.0) as f32; + // Faint texture on both layers. + let tf = (noise(p[0], p[1], 3) as f32 - 0.5) * 0.04; + let tb = (noise(p[0], p[1], 4) as f32 - 0.5) * 0.04; + let f = [fg[0] + tf, fg[1] + tf, fg[2] + tf]; + let b = [bg[0] + tb, bg[1] + tb, bg[2] + tb]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = a; + } + } + (img, gt) +} + +fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke { + Stroke { + kind: StrokeKind::Refine, + frame, + radius, + points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(), + } +} + +#[test] +fn refine_edge_recovers_a_soft_edge_and_decontaminates() { + let c = [100.0, 75.0]; + let (fgc, bgc) = ([0.9f32, 0.25, 0.1], [0.1f32, 0.3, 0.9]); + let (img, gt) = soft_frame(c, 10.0, fgc, bgc); + let mut st = vec![ + line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0), + line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0), + ]; + st.push(ring(0, c, R, 9.0)); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 9.0); + assert!(seg.refine.iter().filter(|v| **v != 0).count() > 1000); + let p = MatteParams::default(); + let rgb = rgb_of(&img); + let a = matte_alpha(&rgb, &seg, None, None, &p, 9.0, 1.0); + // Inside the ramp the matte is fractional and close to the truth. + let (mut err, mut n, mut frac) = (0.0f64, 0usize, 0usize); + let (mut err_bin, mut n_bin) = (0.0f64, 0usize); + for i in 0..a.len() { + if gt[i] > 0.0 && gt[i] < 1.0 { + err += (a[i] - gt[i]).abs() as f64; + err_bin += (seg.matte[i] as f32 - gt[i]).abs() as f64; + n += 1; + n_bin += 1; + frac += (a[i] > 0.05 && a[i] < 0.95) as usize; + } + } + let (err, err_bin) = (err / n as f64, err_bin / n_bin as f64); + assert!(err < 0.03, "mean alpha error {err} (binary {err_bin})"); + assert!(err < err_bin * 0.4); + assert!(frac as f64 > 0.6 * n as f64, "{frac}/{n} fractional"); + // Decontamination: edge pixels' colour is the foreground's, not a blue blend. + let mut col = rgb.clone(); + let map = decontaminate(&mut col, &a, &seg.refine, W as usize, H as usize, &p, 9.0, 1.0); + let (mut blue_before, mut blue_after, mut m) = (0.0f64, 0.0f64, 0usize); + for i in 0..a.len() { + if a[i] > 0.2 && a[i] < 0.8 { + blue_before += rgb[i][2] as f64; + blue_after += col[i][2] as f64; + m += 1; + } + } + let (bb, ba) = (blue_before / m as f64, blue_after / m as f64); + assert!(ba < 0.2 && bb > 0.35, "blue before {bb}, after {ba}"); + assert!(map.iter().any(|v| *v > 0.2)); + // Without the band, the binary matte passes through. + let mut seg2 = seg.clone(); + seg2.refine.iter_mut().for_each(|v| *v = 0); + let a2 = matte_alpha(&rgb, &seg2, None, None, &p, 9.0, 1.0); + assert!(a2.iter().zip(&seg.matte).all(|(x, y)| (*x - *y as f32).abs() < 1e-6)); +} + +#[test] +fn reduce_chatter_and_motion_blur_use_the_neighbours() { + let (w, h) = (40usize, 20usize); + let mk = |x0: usize| FrameSeg { + w, + h, + matte: (0..w * h).map(|i| (i % w >= x0 && i % w < x0 + 10) as u8).collect(), + refine: vec![0; w * h], + ..Default::default() + }; + let (a, b, c) = (mk(10), mk(14), mk(18)); + let rgb = vec![[0.5f32; 3]; w * h]; + let p = MatteParams { rb_chatter: 100.0, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + // Pixel 12 is only in the previous frame: partially on. + assert!(al[12] > 0.2 && al[12] < 0.5, "{}", al[12]); + let p = MatteParams { motion_blur: true, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + assert!(al[14] > 0.3 && al[14] < 0.9, "{}", al[14]); + assert!(al[19] > 0.99); +} + +#[test] +fn chain_keys_restart_at_correction_frames() { + let mut d = RotoData::default(); + for s in base_strokes(5) { + d.add(s, 20, [0, 30]); + } + assert_eq!(d.base, Some(5)); + assert_eq!(d.span, [0, 25]); + let k1 = d.chain_keys(1); + d.add(line(StrokeKind::Fg, 12, [0.0, 0.0], [1.0, 1.0], 3.0), 20, [0, 30]); + let k2 = d.chain_keys(1); + for f in 0..=11 { + assert_eq!(k1[&f], k2[&f], "frame {f}"); + } + for f in 12..=25 { + assert_ne!(k1[&f], k2[&f], "frame {f}"); + } + assert_ne!(d.chain_keys(2)[&5], k2[&5]); + let back = RotoData::from_json(&d.to_json()); + assert_eq!(back, d); + assert_eq!(d.source_of(12), Some(11)); + assert_eq!(d.source_of(3), Some(4)); + assert_eq!(d.source_of(5), None); +} + +/// `cargo test -p effectcraft-track --release roto_perf -- --ignored --nocapture` +#[test] +#[ignore] +fn roto_perf_1080p() { + let up = |c: [f64; 2]| { + let (img, _) = frame_at(c); + effectcraft_raster::resample(&img, 1920, 1080) + }; + let s = 1920.0 / W as f64; + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut k| { + k.points.iter_mut().for_each(|p| *p = [p[0] * s, p[1] * s]); + k.radius *= s; + k + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts { search_radius: 15.0 * s / 4.0, ..Default::default() }; + let a = up(center(0)); + let t = std::time::Instant::now(); + let seg = segment(&a, &refs, None, &opts, 0.0); + eprintln!("segment 1080p: {:?}", t.elapsed()); + let b = up(center(1)); + let t = std::time::Instant::now(); + let next = propagate(&a, &seg, &b, &[], &opts, 0.0); + eprintln!("propagate 1080p: {:?} (area {})", t.elapsed(), next.area()); + let mut seg2 = next.clone(); + let c = center(1); + let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect(); + let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring }]; + let r = rasterize(&st.iter().collect::>(), &[StrokeKind::Refine], 1.0, 1920, 1080); + seg2.refine = r.iter().map(|v| *v as u8).collect(); + let rgb = rgb_of(&b); + let t = std::time::Instant::now(); + let _ = matte_alpha(&rgb, &seg2, None, None, &MatteParams::default(), 20.0, 1.0); + eprintln!("refine 1080p: {:?}", t.elapsed()); +} From 8404fead60a35f3007f6fe1dae80530d48716a21 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:21:08 -0400 Subject: [PATCH 07/14] UI-8, UI-3: native macOS menu bar; timeline columns, search, reveal-add, UU, CTI snapping Native menu bar (apps/effectcraft, muda, macOS only): the engine's After Effects menu tree mirrored into NSMenu through a pure, serializable spec (ui-egui native_menu: submenus, separators, live labels, check marks, enabled state, Cmd/Ctrl/function-key shortcuts as accelerators, Open Recent). Hide / Hide Others / Show All are macOS' predefined items (app.hideOthers/app.showAll are real commands now, via NSApplication); Services added; About, Settings and Quit dispatch our commands. State is pushed only when the session revision, selection, history, settings or journal change; structural changes rebuild. Activations dispatch the same command ids as the in-window bar; Edit Cut/Copy/Paste/Select All/Undo/Redo act on a focused text field. Settings > Appearance > Use In-Window Menu Bar on macOS switches back. Timeline: column visibility from the column header's context menu (A/V Features, Keys, Label, #, Comment, Switches, Modes, Parent & Link, In, Out, Duration, Stretch; Source/Layer Name toggle), Comment editing (layer.setComment), draggable In/Out/Duration cells, Stretch opens Time Stretch; property search filters layers and reveals matching properties with their groups; Shift+P/S/R/T/A/E/M/F/L/U add to the revealed properties; UU reveals all modified properties; Shift-drag in the ruler snaps the CTI to keys, in/out points, markers and the work area. Commands: timeline.column, timeline.sourceName, timeline.search, timeline.revealAdd.*. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 42 ++ apps/effectcraft/Cargo.toml | 6 + apps/effectcraft/src/main.rs | 40 +- apps/effectcraft/src/native_menu.rs | 210 ++++++ crates/engine/src/commands/frontend.rs | 3 + crates/engine/src/commands/layer.rs | 15 + crates/engine/src/commands/stubs.rs | 3 - crates/engine/src/prefs.rs | 4 + crates/ui-egui/src/lib.rs | 6 + crates/ui-egui/src/menus.rs | 113 ++- crates/ui-egui/src/native_menu.rs | 459 ++++++++++++ crates/ui-egui/src/panels/timeline.rs | 931 ++++++++++++++++++++----- crates/ui-egui/src/state.rs | 14 + 13 files changed, 1640 insertions(+), 206 deletions(-) create mode 100644 apps/effectcraft/src/native_menu.rs create mode 100644 crates/ui-egui/src/native_menu.rs diff --git a/Cargo.lock b/Cargo.lock index 871c9fd84..fb72a3038 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,6 +971,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -1133,6 +1142,10 @@ dependencies = [ "effectcraft-ui-egui", "eframe", "egui", + "log", + "muda", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "rfd", "serde_json", @@ -2619,6 +2632,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keyboard-types" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fbe853b403ae61a04233030ae8a79d94975281ed9770a1f9e246732b534b28d" +dependencies = [ + "bitflags 2.13.2", + "serde", +] + [[package]] name = "khronos-egl" version = "6.0.0" @@ -2826,6 +2849,25 @@ dependencies = [ "pxfm", ] +[[package]] +name = "muda" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6744ab645ebe79f9c428455c7cb9dfa4275fa3b50a25408e08f927da8dc618c" +dependencies = [ + "crossbeam-channel", + "dpi", + "keyboard-types", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "png", + "thiserror 2.0.21", + "windows-sys 0.61.2", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 6b34763f2..3d5e1bb12 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -18,6 +18,7 @@ eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", " serde_json = { workspace = true } rfd = { workspace = true } cpal = { workspace = true } +log = { workspace = true } [build-dependencies] # Icon + VERSIONINFO for effectcraft.exe; build.rs only uses it when targeting Windows. @@ -28,3 +29,8 @@ winresource = "0.1" winit = { version = "0.30", default-features = false } # Keep the control channel responsive when the window is covered (App Nap / occlusion throttling). objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSProcessInfo", "NSString", "NSObject"] } +# The native menu bar (NSMenu). MIT OR Apache-2.0; macOS only, never in the web build. +muda = { version = "0.21", default-features = false } +objc2 = "0.6" +# NSApplication hide / hide others / show all, and the pasteboard for Edit ▸ Paste in text fields. +objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSApplication", "NSResponder", "NSPasteboard"] } diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 34b40a84d..b6426f5f0 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -7,6 +7,8 @@ mod audio_out; mod control_server; +#[cfg(target_os = "macos")] +mod native_menu; use effectcraft_ui_egui::EffectcraftApp; use serde_json::json; @@ -100,16 +102,52 @@ fn main() -> eframe::Result { app.hooks.pick_save_file = Some(Box::new(|name: &str, ext: &str| { rfd::FileDialog::new().add_filter(ext, &[ext]).set_file_name(name).save_file().map(|p| p.to_string_lossy().to_string()) })); + #[cfg(target_os = "macos")] + { + app.hooks.app_action = Some(Box::new(native_menu::app_action)); + app.hooks.clipboard_text = Some(Box::new(native_menu::clipboard_text)); + } if let Some(port) = control_port { disable_app_nap(); let rx = control_server::start(port, cc.egui_ctx.clone()); app = app.with_control(rx); } - Ok(Box::new(app)) + Ok(Box::new(Desktop { + #[cfg(target_os = "macos")] + menu: native_menu::NativeBar::new(&cc.egui_ctx), + app, + })) }), ) } +/// The desktop app: the egui app plus, on macOS, the system menu bar. +struct Desktop { + app: EffectcraftApp, + #[cfg(target_os = "macos")] + menu: native_menu::NativeBar, +} + +impl eframe::App for Desktop { + fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { + #[cfg(target_os = "macos")] + self.menu.update(&mut self.app, ctx); + self.app.logic(ctx, frame); + } + + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { + self.app.ui(ui, frame); + } + + fn on_exit(&mut self) { + self.app.on_exit(); + } + + fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { + self.app.raw_input_hook(ctx, raw_input); + } +} + /// The platform config directory for EffectCraft (`EFFECTCRAFT_CONFIG_DIR` overrides): /// `~/Library/Application Support/EffectCraft` (macOS), `%APPDATA%\EffectCraft` (Windows), /// `$XDG_CONFIG_HOME/effectcraft` or `~/.config/effectcraft` (Linux and others). diff --git a/apps/effectcraft/src/native_menu.rs b/apps/effectcraft/src/native_menu.rs new file mode 100644 index 000000000..a20b27f5f --- /dev/null +++ b/apps/effectcraft/src/native_menu.rs @@ -0,0 +1,210 @@ +//! macOS: the menu bar in the system menu bar (`NSMenu` through `muda`), built from +//! [`effectcraft_ui_egui::native_menu`]'s spec of the engine's After Effects menu tree. +//! +//! Each frame [`NativeBar::update`] runs menu activations (the same command ids as the in-window +//! bar), and when [`native_menu::state_key`] changed it rebuilds the spec and pushes labels, +//! enabled and check state into the existing items; a structural change (Open Recent, a new +//! shortcut preset) rebuilds the menu. Settings ▸ Appearance ▸ Use In-Window Menu Bar on macOS +//! swaps back to the in-window bar. + +use std::collections::HashMap; +use std::sync::mpsc::{Receiver, channel}; + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::native_menu::{self, NativeMenu, NativeNode, Role}; +use muda::{CheckMenuItem, Menu, MenuEvent, PredefinedMenuItem, Submenu}; + +/// The `muda` accelerator for a spec accelerator string. +pub fn parse_accelerator(s: &str) -> Option { + s.parse().ok() +} + +pub struct NativeBar { + menu: Option, + items: HashMap, + spec: NativeMenu, + structure: u64, + state: Option, + events: Receiver, + /// The in-window-bar setting last applied (None: not yet). + in_window: Option, +} + +impl NativeBar { + pub fn new(ctx: &egui::Context) -> Self { + let (tx, rx) = channel(); + let ctx = ctx.clone(); + MenuEvent::set_event_handler(Some(move |e: MenuEvent| { + let _ = tx.send(e); + ctx.request_repaint(); + })); + NativeBar { menu: None, items: HashMap::new(), spec: NativeMenu::default(), structure: 0, state: None, events: rx, in_window: None } + } + + /// Run activations and keep the native menu in step with the app (call once per frame). + pub fn update(&mut self, app: &mut EffectcraftApp, ctx: &egui::Context) { + let in_window = app.session.prefs.appearance.in_window_menu_bar_mac; + if self.in_window != Some(in_window) { + self.in_window = Some(in_window); + app.ui.show_menu_bar = in_window; + if in_window { + if let Some(m) = self.menu.take() { + m.remove_for_nsapp(); + } + self.items.clear(); + } else { + self.state = None; + self.structure = 0; + } + } + while let Ok(e) = self.events.try_recv() { + if let Err(err) = native_menu::activate(app, ctx, &self.spec, &e.id.0) { + log::warn!("menu: {err}"); + } + // Check items toggle themselves when clicked: resync with the app. + self.state = None; + } + if in_window { + return; + } + let key = native_menu::state_key(app); + if self.state == Some(key) && self.menu.is_some() { + return; + } + self.state = Some(key); + let spec = native_menu::build(app); + let structure = spec.structure(); + if self.menu.is_none() || structure != self.structure { + self.install(&spec); + self.structure = structure; + } else { + self.refresh(&spec); + } + self.spec = spec; + } + + fn install(&mut self, spec: &NativeMenu) { + if let Some(m) = self.menu.take() { + m.remove_for_nsapp(); + } + self.items.clear(); + let menu = Menu::new(); + for node in &spec.menus { + if let NativeNode::Submenu { label, children } = node { + let sub = Submenu::new(label, true); + self.fill(&sub, children); + match label.as_str() { + "Window" => sub.set_as_windows_menu_for_nsapp(), + "Help" => sub.set_as_help_menu_for_nsapp(), + _ => {} + } + if let Err(e) = menu.append(&sub) { + log::warn!("menu {label}: {e}"); + } + } + } + menu.init_for_nsapp(); + self.menu = Some(menu); + } + + fn fill(&mut self, parent: &Submenu, nodes: &[NativeNode]) { + for n in nodes { + let r = match n { + NativeNode::Separator => parent.append(&PredefinedMenuItem::separator()), + NativeNode::Predefined { role, label, .. } => { + let item = match role { + Role::Services => PredefinedMenuItem::services(Some(label)), + Role::Hide => PredefinedMenuItem::hide(Some(label)), + Role::HideOthers => PredefinedMenuItem::hide_others(Some(label)), + Role::ShowAll => PredefinedMenuItem::show_all(Some(label)), + }; + parent.append(&item) + } + NativeNode::Submenu { label, children } => { + let sub = Submenu::new(label, true); + self.fill(&sub, children); + parent.append(&sub) + } + NativeNode::Item(i) => { + let accel = i.accelerator.as_deref().and_then(parse_accelerator); + let item = CheckMenuItem::with_id(i.id.clone(), &i.label, i.enabled, i.checked == Some(true), accel); + let r = parent.append(&item); + self.items.insert(i.id.clone(), item); + r + } + }; + if let Err(e) = r { + log::warn!("menu item: {e}"); + } + } + } + + /// Push label / enabled / check changes into the existing items. + fn refresh(&self, spec: &NativeMenu) { + let old: HashMap<&str, &native_menu::NativeItem> = self.spec.items().into_iter().map(|i| (i.id.as_str(), i)).collect(); + for i in spec.items() { + let Some(native) = self.items.get(&i.id) else { continue }; + let prev = old.get(i.id.as_str()); + if prev.is_none_or(|p| p.label != i.label) { + native.set_text(&i.label); + } + if prev.is_none_or(|p| p.enabled != i.enabled) { + native.set_enabled(i.enabled); + } + let checked = i.checked == Some(true); + if native.is_checked() != checked { + native.set_checked(checked); + } + } + } +} + +/// `app.hide` / `app.hideOthers` / `app.showAll` through NSApplication (main thread only). +pub fn app_action(id: &str) -> bool { + use objc2::MainThreadMarker; + use objc2_app_kit::NSApplication; + let Some(mtm) = MainThreadMarker::new() else { return false }; + let app = NSApplication::sharedApplication(mtm); + match id { + "app.hide" => app.hide(None), + "app.hideOthers" => app.hideOtherApplications(None), + "app.showAll" => app.unhideAllApplications(None), + _ => return false, + } + true +} + +/// The general pasteboard's text. +pub fn clipboard_text() -> Option { + use objc2_app_kit::NSPasteboard; + // NSPasteboardTypeString's value (the UTI), without touching the extern static. + let ty = objc2_foundation::NSString::from_str("public.utf8-plain-text"); + NSPasteboard::generalPasteboard().stringForType(&ty).map(|s| s.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn every_accelerator_parses_with_muda() { + let mut s = effectcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let app = EffectcraftApp::new(s); + let spec = native_menu::build(&app); + let mut n = 0; + for i in spec.items() { + if let Some(a) = &i.accelerator { + assert!(parse_accelerator(a).is_some(), "{} `{a}` does not parse", i.label); + n += 1; + } + } + assert!(n > 50, "only {n} accelerators"); + // Spot checks against the AE shortcuts. + let accel = |cmd: &str| spec.items().into_iter().find(|i| i.command == cmd).and_then(|i| i.accelerator.clone()); + assert_eq!(accel("file.saveAs").as_deref(), Some("Cmd+Shift+S")); + assert_eq!(accel("comp.new").as_deref(), Some("Cmd+N")); + assert_eq!(parse_accelerator("Cmd+Shift+S"), "CmdOrCtrl+Shift+S".parse().ok()); + } +} diff --git a/crates/engine/src/commands/frontend.rs b/crates/engine/src/commands/frontend.rs index 39107f25e..f0068ae98 100644 --- a/crates/engine/src/commands/frontend.rs +++ b/crates/engine/src/commands/frontend.rs @@ -26,6 +26,9 @@ pub fn specs() -> Vec { ), fe!("app.gpuInfo", "GPU Information...", [], None, "{}", always), fe!("app.hide", "Hide EffectCraft", [], None, "{}", always), + // macOS: the native menu's Hide Others / Show All (the desktop app hides the other apps). + fe!("app.hideOthers", "Hide Others", [], None, "{}", always), + fe!("app.showAll", "Show All", [], None, "{}", always), fe!("app.quit", "Quit EffectCraft", [], Some("Cmd+Q"), "{}", always), fe!("app.commandPalette", "Quick Apply...", ["Edit"], Some("Cmd+Shift+Space"), "{query?}", always), fe!("app.keyboardShortcuts", "Keyboard Shortcuts", ["Edit"], Some("Cmd+Alt+'"), "{}", always), diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index ac7282001..7b44096ef 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -433,6 +433,20 @@ fn rename(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// The Timeline's Comment column. +fn set_comment(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + let text = str_p(p, "comment").ok_or_else(|| bad("layer.setComment", "missing `comment`"))?.to_string(); + s.edit("Layer Comment", None, |proj, _| { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + for l in comp.layers.iter_mut().filter(|l| ids.contains(&l.id)) { + l.comment = text.clone(); + } + Ok(()) + })?; + Ok(Value::Null) +} + fn blend(s: &mut Session, p: &Value) -> Result { let (cid, ids) = layers_p(s, p)?; let mode = str_p(p, "mode").and_then(BlendMode::from_name); @@ -1073,6 +1087,7 @@ pub fn specs() -> Vec { set_switch ), cmd!("layer.rename", "Rename", [], Some("Enter"), "{layer?, name}", has_layers, rename), + cmd!("layer.setComment", "Layer Comment", [], None, "{layers?, comment}", has_layers, set_comment), cmd!("layer.setBlendMode", "Blending Mode", [], None, "{layers?, mode?: Normal|Multiply|Screen|…, step?: ±1}", has_layers, blend), cmd!( "layer.setTrackMatte", diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 8b244bf68..11239de31 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -17,9 +17,6 @@ macro_rules! stub { pub fn specs() -> Vec { vec![ - // macOS application menu items handled by the OS. - stub!("app.hideOthers", "Hide Others", [], "{}"), - stub!("app.showAll", "Show All", [], "{}"), // Render queue / export (export milestone). stub!("file.createProxy", "Create Proxy", [], "{kind: still|movie}"), // Proxies (no proxy model yet). diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index ec06b3f0b..5a2afab8d 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -115,6 +115,8 @@ page!(Appearance { use_label_color_for_tabs: bool = true, cycle_mask_colors: bool = true, use_gradients: bool = true, + /// macOS: draw the menu bar inside the window instead of the system menu bar. + in_window_menu_bar_mac: bool = false, }); page!(Grids { @@ -740,6 +742,8 @@ pub fn pages() -> Vec { s("appearance.useLabelColorForTabs", "Use Label Color for Related Tabs", B, false), s("appearance.cycleMaskColors", "Cycle Mask Colors", B, false), s("appearance.useGradients", "Use Gradients", B, false), + Section("Menu Bar"), + s("appearance.inWindowMenuBarMac", "Use In-Window Menu Bar on macOS", B, true), ], }, Page { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..e903811de 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -14,6 +14,7 @@ pub mod frames; pub mod header; pub mod icons; pub mod menus; +pub mod native_menu; pub mod panels; pub mod state; pub mod theme; @@ -86,6 +87,11 @@ pub struct Hooks { pub pick_folder: Option Option>>, /// Save dialog for other file kinds: (default name, extension). pub pick_save_file: Option Option>>, + /// The system clipboard's text (native menu Edit ▸ Paste into a text field). + pub clipboard_text: Option Option>>, + /// Application actions the OS performs (`app.hide`, `app.hideOthers`, `app.showAll` on + /// macOS). Returns false when the host doesn't handle the id. + pub app_action: Option bool>>, } #[derive(Default)] diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 22b2bd401..4c85162e5 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -59,6 +59,19 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("timeline.reveal.feather", "Reveal Mask Feather", [], Some("F")), uic!("timeline.reveal.levels", "Reveal Audio Levels (press twice: Waveform)", [], Some("L")), uic!("timeline.reveal.waveform", "Reveal Audio Waveform", [], None), + uic!("timeline.revealAdd.position", "Add Position to Revealed Properties", [], Some("Shift+P")), + uic!("timeline.revealAdd.scale", "Add Scale to Revealed Properties", [], Some("Shift+S")), + uic!("timeline.revealAdd.rotation", "Add Rotation to Revealed Properties", [], Some("Shift+R")), + uic!("timeline.revealAdd.opacity", "Add Opacity to Revealed Properties", [], Some("Shift+T")), + uic!("timeline.revealAdd.anchor", "Add Anchor Point to Revealed Properties", [], Some("Shift+A")), + uic!("timeline.revealAdd.effects", "Add Effects to Revealed Properties", [], Some("Shift+E")), + uic!("timeline.revealAdd.masks", "Add Masks to Revealed Properties", [], Some("Shift+M")), + uic!("timeline.revealAdd.feather", "Add Mask Feather to Revealed Properties", [], Some("Shift+F")), + uic!("timeline.revealAdd.levels", "Add Audio Levels to Revealed Properties", [], Some("Shift+L")), + uic!("timeline.revealAdd.animated", "Add Animated Properties to Revealed Properties", [], Some("Shift+U")), + uic!("timeline.column", "Show/Hide Timeline Column", [], None), + uic!("timeline.sourceName", "Source Name / Layer Name", [], None), + uic!("timeline.search", "Search Timeline", [], None), uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")), uic!("tool.selection", "Selection Tool", [], Some("V")), uic!("tool.hand", "Hand Tool", [], Some("H")), @@ -89,26 +102,42 @@ pub fn panel_command_id(p: PanelKind) -> String { format!("window.panel.{}", p.id()) } -fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64) { +/// The reveal shortcuts (P, S, R, T, A, E, M, F, L…). `add` (Shift+key) adds the property to +/// (or removes it from) those already revealed instead of replacing them. +pub fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64, add: bool) { // AE's double-press shortcuts: L then L again quickly reveals the Waveform (LL). let kind = match app.last_reveal.take() { Some((k, t)) if k == "levels" && kind == "levels" && now - t < 0.6 => "waveform", _ => kind, }; app.last_reveal = Some((kind.to_string(), now)); - let add = app.ui.timeline.reveal.contains(&kind.to_string()); - if add && app.ui.timeline.reveal.len() == 1 { - app.ui.timeline.reveal.clear(); - app.ui.timeline.open_layers.clear(); + let tl = &mut app.ui.timeline; + let present = tl.reveal.iter().any(|k| k == kind); + let sel: Vec = if app.session.state.selected_layers.is_empty() { + app.session.active_comp().map(|c| c.layers.iter().map(|l| l.id.0).collect()).unwrap_or_default() + } else { + app.session.state.selected_layers.iter().map(|l| l.0).collect() + }; + if add { + if present { + tl.reveal.retain(|k| k != kind); + if tl.reveal.is_empty() { + tl.open_layers.clear(); + } + } else { + tl.reveal.retain(|k| k != "props" || !tl.reveal_props.is_empty()); + tl.reveal.push(kind.to_string()); + tl.open_layers.extend(sel); + } return; } - app.ui.timeline.reveal = vec![kind.to_string()]; - let sel = if app.session.state.selected_layers.is_empty() { - app.session.active_comp().map(|c| c.layers.iter().map(|l| l.id).collect()).unwrap_or_default() - } else { - app.session.state.selected_layers.clone() - }; - app.ui.timeline.open_layers = sel.iter().map(|l| l.0).collect(); + if present && tl.reveal.len() == 1 { + tl.reveal.clear(); + tl.open_layers.clear(); + return; + } + tl.reveal = vec![kind.to_string()]; + tl.open_layers = sel.into_iter().collect(); } fn no_params(p: &Value) -> bool { @@ -166,9 +195,30 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V if k == "animated" { return run_engine(app, ctx, "anim.reveal", json!({"kind": "keyframes"})); } - reveal(app, k, now); + reveal(app, k, now, false); return Ok(Value::Null); } + if let Some(k) = id.strip_prefix("timeline.revealAdd.") { + reveal(app, k, now, true); + return Ok(json!({"revealed": app.ui.timeline.reveal})); + } + // U reveals keyframed properties; UU (pressed twice quickly) all modified properties; U + // again later hides them. + if id == "anim.reveal" && params.get("kind").and_then(Value::as_str) == Some("keyframes") { + let last = app.last_reveal.take(); + app.last_reveal = Some(("keyframes".into(), now)); + match last { + Some((k, t)) if k == "keyframes" && now - t < 0.6 => { + return run_engine(app, ctx, "anim.reveal", json!({"kind": "modified"})); + } + Some((k, _)) if k == "keyframes" && app.ui.timeline.reveal == ["props"] => { + app.ui.timeline.reveal.clear(); + app.ui.timeline.open_layers.clear(); + return Ok(json!({"revealed": 0})); + } + _ => {} + } + } // Docking: {panel, anchor, zone: center|left|right|top|bottom}, {panel, rect?}, {panel?}. if let Some(op) = id.strip_prefix("window.").filter(|o| matches!(*o, "maximizePanel" | "dockPanel" | "floatPanel" | "closePanel")) { let panel_of = |k: &str| params.get(k).and_then(Value::as_str).map(|s| PanelKind::from_name(s).ok_or(format!("unknown panel `{s}`"))); @@ -268,6 +318,23 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V return Ok(Value::Null); } "timeline.zoomFit" => app.ui.timeline.pps = None, + "timeline.column" => { + let col = params.get("column").and_then(Value::as_str).ok_or("timeline.column: need `column`")?; + let on = params.get("visible").and_then(Value::as_bool).unwrap_or(!crate::panels::timeline::column_visible(&app.ui.timeline, col)); + crate::panels::timeline::set_column(&mut app.ui.timeline, col, on)?; + let shown: Vec<&str> = + crate::panels::timeline::COLUMNS.iter().map(|c| c.0).filter(|c| crate::panels::timeline::column_visible(&app.ui.timeline, c)).collect(); + return Ok(json!({"columns": shown})); + } + "timeline.sourceName" => { + let tl = &mut app.ui.timeline; + tl.source_name = params.get("value").and_then(Value::as_bool).unwrap_or(!tl.source_name); + return Ok(json!({"sourceName": tl.source_name})); + } + "timeline.search" => { + app.ui.timeline.search = params.get("query").and_then(Value::as_str).unwrap_or_default().to_string(); + app.show_panel(PanelKind::Timeline); + } "timeline.graphEditor" => app.ui.timeline.graph_editor = !app.ui.timeline.graph_editor, "timeline.switchesModes" => app.ui.timeline.show_modes = !app.ui.timeline.show_modes, "timeline.collapseAll" => { @@ -399,8 +466,16 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu ); Value::Null } - "app.hide" => { - ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)); + "app.hide" | "app.hideOthers" | "app.showAll" => { + // The desktop app asks macOS (NSApplication hide / hideOtherApplications / + // unhideAllApplications); elsewhere Hide minimizes and the others don't apply. + if app.hooks.app_action.as_ref().is_some_and(|f| f(id)) { + return Ok(Value::Null); + } + match id { + "app.hide" => ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)), + _ => return Err(format!("`{id}` is only available on macOS")), + } Value::Null } "app.quit" => { @@ -768,7 +843,7 @@ pub fn menus() -> Vec<&'static str> { effectcraft_engine::menus::top_level() } -fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { +pub(crate) fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { match e.command.as_str() { "edit.undo" => app.session.history.undo.last().map(|u| format!("Undo {}", u.0)).unwrap_or_else(|| "Can't Undo".into()), "edit.redo" => app.session.history.redo.last().map(|u| format!("Redo {}", u.0)).unwrap_or_else(|| "Can't Redo".into()), @@ -777,12 +852,12 @@ fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { } /// The entry's shortcut in the active keyboard shortcut preset. -fn entry_shortcut(app: &EffectcraftApp, e: &MenuEntry) -> Option { +pub(crate) fn entry_shortcut(app: &EffectcraftApp, e: &MenuEntry) -> Option { app.session.shortcuts().shortcut_of(&e.command, &e.params).map(str::to_string) } /// Check-mark state of frontend toggles (engine state is answered by the engine). -fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { +pub(crate) fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { let v = &app.ui.viewer; let pstr = |k: &str| e.params.get(k).and_then(Value::as_str); match e.command.as_str() { @@ -814,7 +889,7 @@ fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { } } -fn entry_enabled(app: &EffectcraftApp, e: &MenuEntry) -> bool { +pub(crate) fn entry_enabled(app: &EffectcraftApp, e: &MenuEntry) -> bool { if e.command == "window.panel" { return true; } diff --git a/crates/ui-egui/src/native_menu.rs b/crates/ui-egui/src/native_menu.rs new file mode 100644 index 000000000..ef5070b36 --- /dev/null +++ b/crates/ui-egui/src/native_menu.rs @@ -0,0 +1,459 @@ +//! The native (macOS) menu bar as a plain, serializable spec. The desktop app turns it into an +//! `NSMenu` with `muda` (`apps/effectcraft/src/native_menu.rs`); everything that decides *what* +//! the menu contains lives here, so it is tested without creating real menus and builds for the +//! web (where the in-window menu bar is used). +//! +//! The spec mirrors the engine's After Effects menu tree ([`effectcraft_engine::menus`]): +//! submenus, separators, labels (Undo/Redo and label names are live), check marks, enabled +//! state, and the active shortcut preset's keys as accelerators. File ▸ Open Recent lists the +//! recent projects. The application menu's Hide / Hide Others / Show All are macOS' +//! predefined items, and a Services submenu is added as on every Mac app. +//! +//! Item ids are positional (`m.3.0.2`), stable while the menu's structure stays the same; the +//! app rebuilds the native menu when [`NativeMenu::structure`] changes and otherwise only pushes +//! label / enabled / checked updates, and only when [`state_key`] changes. + +use std::hash::{Hash, Hasher}; + +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use effectcraft_engine::menus::{MenuNode, menu_bar}; + +/// What macOS performs itself (no command dispatch). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Role { + Services, + Hide, + HideOthers, + ShowAll, +} + +/// One node of the native menu. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum NativeNode { + Item(NativeItem), + Separator, + Submenu { + label: String, + children: Vec, + }, + /// A predefined macOS item; `command` is the engine entry it stands for (empty for Services). + Predefined { + role: Role, + label: String, + command: String, + }, +} + +/// A clickable item: activating it runs `command` with `params`, exactly like the in-window bar. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct NativeItem { + pub id: String, + pub label: String, + pub command: String, + #[serde(skip_serializing_if = "Value::is_null")] + pub params: Value, + /// The shortcut in EffectCraft notation (`Cmd+Shift+S`). + pub shortcut: Option, + /// The menu key equivalent in `muda` accelerator notation, when the shortcut can be one (see + /// [`accelerator`]). + pub accelerator: Option, + pub enabled: bool, + pub checked: Option, +} + +/// The whole menu bar: top-level menus in order (the application menu first). +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +pub struct NativeMenu { + pub menus: Vec, +} + +impl NativeMenu { + /// Every clickable item, depth first. + pub fn items(&self) -> Vec<&NativeItem> { + fn rec<'a>(n: &'a [NativeNode], out: &mut Vec<&'a NativeItem>) { + for x in n { + match x { + NativeNode::Item(i) => out.push(i), + NativeNode::Submenu { children, .. } => rec(children, out), + _ => {} + } + } + } + let mut out = vec![]; + rec(&self.menus, &mut out); + out + } + + pub fn find(&self, id: &str) -> Option<&NativeItem> { + self.items().into_iter().find(|i| i.id == id) + } + + /// Fingerprint of everything that needs a rebuild when it changes (tree shape, commands, + /// accelerators, submenu labels); labels, enabled and check state are updated in place. + pub fn structure(&self) -> u64 { + fn rec(n: &[NativeNode], h: &mut std::collections::hash_map::DefaultHasher) { + for x in n { + match x { + NativeNode::Item(i) => { + 0u8.hash(h); + i.id.hash(h); + i.command.hash(h); + i.params.to_string().hash(h); + i.accelerator.hash(h); + } + NativeNode::Separator => 1u8.hash(h), + NativeNode::Submenu { label, children } => { + 2u8.hash(h); + label.hash(h); + rec(children, h); + 3u8.hash(h); + } + NativeNode::Predefined { role, .. } => { + 4u8.hash(h); + role.hash(h); + } + } + } + } + let mut h = std::collections::hash_map::DefaultHasher::new(); + rec(&self.menus, &mut h); + h.finish() + } +} + +/// Build the native menu spec for the app's current state. +pub fn build(app: &EffectcraftApp) -> NativeMenu { + fn rec(app: &EffectcraftApp, nodes: &[MenuNode], path: &str, out: &mut Vec) { + let mut services_added = false; + for (i, n) in nodes.iter().enumerate() { + let id = format!("{path}.{i}"); + match n { + MenuNode::Separator => out.push(NativeNode::Separator), + MenuNode::Submenu { label, children } => { + let mut kids = vec![]; + if label == "Open Recent" { + for (k, p) in app.session.prefs.recent_projects.iter().enumerate() { + let name = std::path::Path::new(p).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(p.clone()); + kids.push(NativeNode::Item(NativeItem { + id: format!("{id}.r{k}"), + label: name, + command: "file.openRecent".into(), + params: json!({"index": k}), + shortcut: None, + accelerator: None, + enabled: true, + checked: None, + })); + } + if !kids.is_empty() { + kids.push(NativeNode::Separator); + } + } + rec(app, children, &id, &mut kids); + out.push(NativeNode::Submenu { label: label.clone(), children: kids }); + } + MenuNode::Item(e) => { + let role = match e.command.as_str() { + "app.hide" => Some(Role::Hide), + "app.hideOthers" => Some(Role::HideOthers), + "app.showAll" => Some(Role::ShowAll), + _ => None, + }; + if let Some(role) = role { + // Services sits just above Hide, as in every Mac app. + if role == Role::Hide && !services_added { + out.push(NativeNode::Predefined { role: Role::Services, label: "Services".into(), command: String::new() }); + out.push(NativeNode::Separator); + services_added = true; + } + out.push(NativeNode::Predefined { role, label: e.label.clone(), command: e.command.clone() }); + continue; + } + let shortcut = crate::menus::entry_shortcut(app, e); + out.push(NativeNode::Item(NativeItem { + id, + label: crate::menus::entry_label(app, e), + command: e.command.clone(), + params: e.params.clone(), + accelerator: shortcut.as_deref().and_then(accelerator), + shortcut, + enabled: crate::menus::entry_enabled(app, e), + checked: crate::menus::entry_checked(app, e), + })); + } + } + } + } + let mut menus = vec![]; + rec(app, menu_bar(), "m", &mut menus); + NativeMenu { menus } +} + +/// The `muda` accelerator for an EffectCraft shortcut, or `None` when it must stay a plain +/// keyboard shortcut. Only shortcuts with Cmd or Ctrl (and function keys) become menu key +/// equivalents: a single-key shortcut (`V`, `Space`, `U`) or an Alt/Shift-only one would be +/// swallowed by the menu while typing into a text field. Those still work through the +/// window's shortcut dispatcher, as on the other platforms. +pub fn accelerator(shortcut: &str) -> Option { + let parts: Vec<&str> = match shortcut.strip_suffix("++") { + Some(head) => head.split('+').chain(std::iter::once("+")).collect(), + None => shortcut.split('+').collect(), + }; + let (key, mods) = parts.split_last()?; + let mut out: Vec = vec![]; + let mut primary = false; + for m in mods { + out.push( + match *m { + "Cmd" => { + primary = true; + "Cmd" + } + "Ctrl" => { + primary = true; + "Ctrl" + } + "Alt" => "Alt", + "Shift" => "Shift", + _ => return None, + } + .to_string(), + ); + } + let function_key = key.len() >= 2 && key.starts_with('F') && key[1..].parse::().is_ok_and(|n| (1..=24).contains(&n)); + if !primary && !function_key { + return None; + } + let k = match *key { + "Num*" => "NumpadMultiply".to_string(), + "+" => "Equal".to_string(), + k if k.starts_with("Num") && k.len() == 4 => format!("Numpad{}", &k[3..]), + "Escape" | "Space" | "Enter" | "Tab" | "Home" | "End" | "PageUp" | "PageDown" | "Delete" | "Backspace" => key.to_string(), + "ArrowLeft" | "ArrowRight" | "ArrowUp" | "ArrowDown" => key.to_string(), + k if k.chars().count() == 1 => { + let c = k.chars().next()?; + if c.is_ascii_alphanumeric() || "`\\[],-=./;'".contains(c) { + c.to_ascii_uppercase().to_string() + } else { + return None; + } + } + k if function_key => k.to_string(), + _ => return None, + }; + out.push(k); + Some(out.join("+")) +} + +/// Cheap per-frame fingerprint of everything menu labels, check marks and enabled state depend +/// on (project revision, selection, history, settings, executed commands, viewer toggles). +pub fn state_key(app: &EffectcraftApp) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + let s = &app.session; + s.revision.hash(&mut h); + s.prefs_revision.hash(&mut h); + s.path.hash(&mut h); + s.active_comp_id().map(|c| c.0).hash(&mut h); + for l in &s.state.selected_layers { + l.0.hash(&mut h); + } + s.state.selected_keys.len().hash(&mut h); + s.state.selected_props.len().hash(&mut h); + s.state.project_selection.len().hash(&mut h); + s.state.selected_vertices.len().hash(&mut h); + s.state.clipboard.len().hash(&mut h); + s.state.key_clipboard.len().hash(&mut h); + (s.history.undo.len(), s.history.redo.len()).hash(&mut h); + s.journal.len().hash(&mut h); + s.journal.last().map(|j| &j.0).hash(&mut h); + s.prefs.recent_projects.hash(&mut h); + s.keymaps.active.hash(&mut h); + let v = &app.ui.viewer; + (v.rulers, v.guides, v.snap_guides, v.lock_guides, v.grid, v.snap_grid, v.show_layer_controls, v.pasteboard).hash(&mut h); + v.res.label().hash(&mut h); + (app.ui.cache_when_idle, app.ui.preview_audio, &app.ui.workspace, app.dialog.is_some()).hash(&mut h); + h.finish() +} + +/// Commands that act on a focused text field instead (Edit menu while typing): the egui event to +/// send in their place. +fn text_field_event(command: &str, clipboard: Option) -> Option> { + let key = |k: egui::Key, shift: bool| egui::Event::Key { + key: k, + physical_key: None, + pressed: true, + repeat: false, + modifiers: egui::Modifiers { shift, ..egui::Modifiers::COMMAND }, + }; + Some(match command { + "edit.copy" => vec![egui::Event::Copy], + "edit.cut" => vec![egui::Event::Cut], + "edit.paste" => vec![egui::Event::Paste(clipboard?)], + "edit.selectAll" => vec![key(egui::Key::A, false)], + "edit.undo" => vec![key(egui::Key::Z, false)], + "edit.redo" => vec![key(egui::Key::Z, true)], + _ => return None, + }) +} + +/// Run a native menu item (by id) the way the in-window menu bar does. While a text field has +/// the keyboard, Edit ▸ Cut/Copy/Paste/Select All/Undo/Redo edit the text instead. +pub fn activate(app: &mut EffectcraftApp, ctx: &egui::Context, menu: &NativeMenu, id: &str) -> Result { + let item = menu.find(id).ok_or_else(|| format!("no native menu item `{id}`"))?; + let (command, params) = (item.command.clone(), item.params.clone()); + if ctx.egui_wants_keyboard_input() { + let clip = if command == "edit.paste" { app.hooks.clipboard_text.as_ref().and_then(|f| f()) } else { None }; + if let Some(events) = text_field_event(&command, clip) { + app.synthetic.extend(events); + ctx.request_repaint(); + return Ok(Value::Null); + } + } + let params = if params.is_null() { json!({}) } else { params }; + let r = crate::menus::invoke(app, ctx, &command, params); + if let Err(e) = &r { + app.ui.status = e.clone(); + } + ctx.request_repaint(); + r +} + +#[cfg(test)] +mod tests { + use super::*; + + fn app() -> EffectcraftApp { + let mut s = effectcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + EffectcraftApp::new(s) + } + + #[test] + fn every_engine_entry_is_in_the_native_menu() { + let app = app(); + let spec = build(&app); + fn commands(n: &[NativeNode], out: &mut Vec<(String, String)>) { + for x in n { + match x { + NativeNode::Item(i) if i.command != "file.openRecent" => out.push((i.command.clone(), i.params.to_string())), + NativeNode::Predefined { command, .. } if !command.is_empty() => out.push((command.clone(), "null".into())), + NativeNode::Submenu { children, .. } => commands(children, out), + _ => {} + } + } + } + let mut got = vec![]; + commands(&spec.menus, &mut got); + let want: Vec<(String, String)> = effectcraft_engine::menus::entries().into_iter().map(|(_, e)| (e.command.clone(), e.params.to_string())).collect(); + assert_eq!(got, want, "native menu must mirror the engine tree entry for entry"); + // Top-level menus in order. + let tops: Vec<&str> = spec + .menus + .iter() + .filter_map(|n| match n { + NativeNode::Submenu { label, .. } => Some(label.as_str()), + _ => None, + }) + .collect(); + assert_eq!(tops, effectcraft_engine::menus::top_level()); + // Ids are unique. + let ids: std::collections::BTreeSet<&str> = spec.items().iter().map(|i| i.id.as_str()).collect(); + assert_eq!(ids.len(), spec.items().len()); + // The spec serializes (agents can read it). + let v = serde_json::to_value(&spec).unwrap(); + assert!(v["menus"].as_array().is_some_and(|a| a.len() >= 9)); + } + + #[test] + fn hide_items_are_predefined_on_macos() { + let spec = build(&app()); + let mut roles = vec![]; + fn rec(n: &[NativeNode], out: &mut Vec) { + for x in n { + match x { + NativeNode::Predefined { role, .. } => out.push(*role), + NativeNode::Submenu { children, .. } => rec(children, out), + _ => {} + } + } + } + rec(&spec.menus, &mut roles); + if cfg!(target_os = "macos") { + assert_eq!(roles, vec![Role::Services, Role::Hide, Role::HideOthers, Role::ShowAll]); + // Quit and About dispatch our commands (clean shutdown, our About dialog). + assert!(spec.items().iter().any(|i| i.command == "app.quit" && i.accelerator.as_deref() == Some("Cmd+Q"))); + assert!(spec.items().iter().any(|i| i.command == "app.about")); + } else { + assert!(roles.is_empty()); + } + } + + #[test] + fn accelerators_convert() { + assert_eq!(accelerator("Cmd+Shift+S").as_deref(), Some("Cmd+Shift+S")); + assert_eq!(accelerator("Cmd+Alt+;").as_deref(), Some("Cmd+Alt+;")); + assert_eq!(accelerator("Cmd+`").as_deref(), Some("Cmd+`")); + assert_eq!(accelerator("Ctrl+Cmd+F").as_deref(), Some("Ctrl+Cmd+F")); + assert_eq!(accelerator("F3").as_deref(), Some("F3")); + assert_eq!(accelerator("Cmd+Alt+0").as_deref(), Some("Cmd+Alt+0")); + // Keys that would swallow typing stay window shortcuts. + assert_eq!(accelerator("V"), None); + assert_eq!(accelerator("Space"), None); + assert_eq!(accelerator("Shift+="), None); + assert_eq!(accelerator("Alt+["), None); + assert_eq!(accelerator("Num*"), None); + // Every menu shortcut either converts or is a single-key/Alt/Shift shortcut. + let app = app(); + for i in build(&app).items() { + if let Some(sc) = &i.shortcut { + let primary = sc.contains("Cmd") || sc.contains("Ctrl"); + if primary { + assert!(i.accelerator.is_some(), "{} ({sc}) has no accelerator", i.label); + } + } + } + } + + #[test] + fn state_changes_flow_into_the_spec() { + let mut app = app(); + let k0 = state_key(&app); + let spec0 = build(&app); + let undo = |s: &NativeMenu| s.items().into_iter().find(|i| i.command == "edit.undo").map(|i| (i.label.clone(), i.enabled)).unwrap(); + // Select a layer and toggle Lock: the key changes, the Lock entry gets a check mark. + let first = app.session.active_comp().unwrap().layers[0].id.0; + app.session.execute("layer.select", json!({"layers": [first]})).unwrap(); + app.session.execute("layer.setSwitch", json!({"switch": "lock"})).unwrap(); + assert_ne!(state_key(&app), k0); + let spec1 = build(&app); + assert_eq!(spec0.structure(), spec1.structure(), "state changes don't rebuild the menu"); + let lock = spec1.items().into_iter().find(|i| i.command == "layer.setSwitch" && i.params["switch"] == "lock").unwrap().checked; + assert_eq!(lock, Some(true)); + assert_ne!(undo(&spec0), undo(&spec1)); + // A recent project changes the structure (Open Recent grows). + app.session.prefs.push_recent("/tmp/Recent Test.ecproj"); + let spec2 = build(&app); + assert_ne!(spec1.structure(), spec2.structure()); + assert!(spec2.items().iter().any(|i| i.command == "file.openRecent" && i.label == "Recent Test.ecproj")); + } + + #[test] + fn activation_dispatches_the_menu_command() { + let mut app = app(); + let ctx = egui::Context::default(); + let spec = build(&app); + let id = spec.items().into_iter().find(|i| i.command == "layer.newNull").unwrap().id.clone(); + let n = app.session.active_comp().unwrap().layers.len(); + activate(&mut app, &ctx, &spec, &id).unwrap(); + assert_eq!(app.session.active_comp().unwrap().layers.len(), n + 1); + assert!(activate(&mut app, &ctx, &spec, "m.999").is_err()); + assert!(text_field_event("edit.copy", None).is_some()); + assert!(text_field_event("edit.paste", None).is_none()); + assert!(text_field_event("layer.newNull", None).is_none()); + } +} diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index c9a42698a..b9289e77b 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -93,30 +93,207 @@ const LABEL_W: f32 = 20.0; const NUM_W: f32 = 26.0; const SW: f32 = 19.0; +/// The Timeline's columns in After Effects' order: (id, header label). Right-click a column +/// header to show or hide them; the name column is always shown. +pub const COLUMNS: [(&str, &str); 13] = [ + ("av", "A/V Features"), + ("keys", "Keys"), + ("label", "Label"), + ("num", "#"), + ("name", "Layer Name"), + ("comment", "Comment"), + ("switches", "Switches"), + ("modes", "Modes"), + ("parent", "Parent & Link"), + ("in", "In"), + ("out", "Out"), + ("duration", "Duration"), + ("stretch", "Stretch"), +]; + +/// Is a column shown? +pub fn column_visible(tl: &crate::state::TimelineState, id: &str) -> bool { + match id { + "name" => true, + "modes" => tl.show_modes, + _ => tl.columns.contains(id), + } +} + +/// Show or hide a column (`modes` is the Switches/Modes toggle, F4). +pub fn set_column(tl: &mut crate::state::TimelineState, id: &str, on: bool) -> Result<(), String> { + match id { + "name" if on => Ok(()), + "name" => Err("the name column can't be hidden".into()), + "modes" => { + tl.show_modes = on; + Ok(()) + } + _ if COLUMNS.iter().any(|c| c.0 == id) => { + if on { + tl.columns.insert(id.to_string()); + } else { + tl.columns.remove(id); + } + Ok(()) + } + _ => Err(format!("unknown column `{id}` (one of: {})", COLUMNS.map(|c| c.0).join(", "))), + } +} + +const KEYS_W: f32 = 56.0; +const COMMENT_W: f32 = 120.0; +const TIME_W: f32 = 72.0; +const STRETCH_W: f32 = 60.0; +const PARENT_W: f32 = 116.0; +const MODES_W: f32 = 92.0 + 112.0; + +#[derive(Clone, Copy, Default)] +struct Vis { + av: bool, + keys: bool, + label: bool, + num: bool, + comment: bool, + switches: bool, + modes: bool, + parent: bool, + in_: bool, + out: bool, + duration: bool, + stretch: bool, +} + +impl Vis { + fn of(tl: &crate::state::TimelineState) -> Vis { + let v = |id| column_visible(tl, id); + Vis { + av: v("av"), + keys: v("keys"), + label: v("label"), + num: v("num"), + comment: v("comment"), + switches: v("switches"), + modes: v("modes"), + parent: v("parent"), + in_: v("in"), + out: v("out"), + duration: v("duration"), + stretch: v("stretch"), + } + } + /// Width of every visible column but the name. + fn fixed(&self) -> f32 { + let w = |on: bool, w: f32| if on { w } else { 0.0 }; + w(self.av, AV_W) + + w(self.keys, KEYS_W) + + w(self.label, LABEL_W) + + w(self.num, NUM_W) + + w(self.comment, COMMENT_W) + + w(self.switches, SW * 8.0 + 6.0) + + w(self.modes, MODES_W) + + w(self.parent, PARENT_W) + + w(self.in_, TIME_W) + + w(self.out, TIME_W) + + w(self.duration, TIME_W) + + w(self.stretch, STRETCH_W) + } +} + struct Cols { + vis: Vis, av: f32, + keys: f32, label: f32, num: f32, name: f32, + /// End of the name column. + name_end: f32, + comment: f32, switches: f32, mode: f32, trkmat: f32, parent: f32, + in_: f32, + out: f32, + duration: f32, + stretch: f32, end: f32, } -fn cols(x0: f32, width: f32, show_modes: bool) -> Cols { - let fixed = AV_W + LABEL_W + NUM_W + SW * 8.0 + if show_modes { 92.0 + 112.0 } else { 0.0 } + 116.0; - let name_w = (width - fixed).max(120.0); - let av = x0; - let label = av + AV_W; - let num = label + LABEL_W; - let name = num + NUM_W; - let switches = name + name_w; - let mode = switches + SW * 8.0 + 6.0; - let trkmat = mode + if show_modes { 92.0 } else { 0.0 }; - let parent = trkmat + if show_modes { 112.0 } else { 0.0 }; - Cols { av, label, num, name, switches, mode, trkmat, parent, end: parent + 116.0 } +fn cols(x0: f32, width: f32, vis: Vis) -> Cols { + let name_w = (width - vis.fixed()).max(120.0); + let mut x = x0; + let mut next = |on: bool, w: f32| { + let at = x; + if on { + x += w; + } + at + }; + let av = next(vis.av, AV_W); + let keys = next(vis.keys, KEYS_W); + let label = next(vis.label, LABEL_W); + let num = next(vis.num, NUM_W); + let name = next(true, name_w); + let comment = next(vis.comment, COMMENT_W); + let switches = next(vis.switches, SW * 8.0 + 6.0); + let mode = next(vis.modes, 92.0); + let trkmat = next(vis.modes, 112.0); + let parent = next(vis.parent, PARENT_W); + let in_ = next(vis.in_, TIME_W); + let out = next(vis.out, TIME_W); + let duration = next(vis.duration, TIME_W); + let stretch = next(vis.stretch, STRETCH_W); + let end = next(true, 0.0); + Cols { vis, av, keys, label, num, name, name_end: name + name_w, comment, switches, mode, trkmat, parent, in_, out, duration, stretch, end } +} + +/// Which column a header x position falls in. +fn column_at(cw: &Cols, x: f32) -> Option<&'static str> { + let v = cw.vis; + let spans = [ + ("av", v.av, cw.av, cw.av + AV_W), + ("keys", v.keys, cw.keys, cw.keys + KEYS_W), + ("label", v.label, cw.label, cw.label + LABEL_W), + ("num", v.num, cw.num, cw.num + NUM_W), + ("name", true, cw.name, cw.name_end), + ("comment", v.comment, cw.comment, cw.comment + COMMENT_W), + ("switches", v.switches, cw.switches, cw.switches + SW * 8.0 + 6.0), + ("modes", v.modes, cw.mode, cw.mode + MODES_W), + ("parent", v.parent, cw.parent, cw.parent + PARENT_W), + ("in", v.in_, cw.in_, cw.in_ + TIME_W), + ("out", v.out, cw.out, cw.out + TIME_W), + ("duration", v.duration, cw.duration, cw.duration + TIME_W), + ("stretch", v.stretch, cw.stretch, cw.stretch + STRETCH_W), + ]; + spans.into_iter().find(|(_, on, a, b)| *on && x >= *a && x < *b).map(|s| s.0) +} + +/// Snap a time to the nearest candidate within `tolerance` (all seconds); Shift-dragging the +/// current-time indicator snaps to keyframes, layer in/out points, markers and the work area. +pub fn snap_time(candidates: &[f64], t: f64, tolerance: f64) -> f64 { + candidates.iter().copied().filter(|c| (c - t).abs() <= tolerance).min_by(|a, b| (a - t).abs().total_cmp(&(b - t).abs())).unwrap_or(t) +} + +/// Times the current-time indicator snaps to (comp seconds). +fn snap_candidates(comp: &Comp, rows: &[Row]) -> Vec { + let mut v = vec![0.0, comp.work_area.0.seconds(), comp.work_area.1.seconds(), comp.duration.seconds()]; + v.extend(comp.markers.iter().map(|m| m.time.seconds())); + for l in &comp.layers { + v.push(l.in_point.seconds()); + v.push(l.out_point.seconds()); + v.extend(l.markers.iter().map(|m| l.comp_time(m.time).seconds())); + } + for r in rows { + if let RowKind::Prop { uid } = r.kind + && let Some(l) = comp.layer(r.layer) + && let Some(p) = l.props.find(uid) + { + v.extend(p.keys.iter().map(|k| l.comp_time(k.time).seconds())); + } + } + v } /// Timeline horizontal mapping. @@ -222,127 +399,106 @@ fn reveal_matches(p: &Property, path_matches: &[&str], kind: &str) -> bool { } } +/// The name shown for a layer: its name, or its source's name (Source Name column). +fn display_name(app: &EffectcraftApp, l: &Layer) -> String { + if app.ui.timeline.source_name + && let Some(item) = l.source.item().and_then(|i| app.session.project.item(i)) + { + return item.name.clone(); + } + l.name.clone() +} + +/// The timeline search (After Effects' search field): rows for the properties and groups of a +/// layer whose names contain `q` (lowercase), each with its enclosing groups. A matching group +/// brings its whole contents. +fn search_rows(l: &Layer, q: &str) -> Vec { + fn group_row(l: &Layer, g: &PropGroup, depth: usize, parent: &PropGroup) -> Row { + let fx = matches!(g.kind, GroupKind::Effect { .. }).then_some(g.enabled); + let eye = + (parent.match_id == effectcraft_engine::project::styles::GROUP && g.match_id != effectcraft_engine::project::styles::BLENDING).then_some(g.enabled); + Row { layer: l.id, depth, kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open: true, has_children: !g.children.is_empty(), fx, eye } } + } + fn all(l: &Layer, g: &PropGroup, depth: usize, out: &mut Vec) { + for c in &g.children { + match c { + Node::Group(sg) => { + out.push(group_row(l, sg, depth, g)); + all(l, sg, depth + 1, out); + } + Node::Prop(p) if prop_visible(p, l) => out.push(Row { layer: l.id, depth, kind: RowKind::Prop { uid: p.uid } }), + _ => {} + } + } + } + fn rec(l: &Layer, g: &PropGroup, depth: usize, q: &str, out: &mut Vec) { + for c in &g.children { + match c { + Node::Group(sg) if depth > 1 || group_visible(sg, l) => { + if sg.name.to_lowercase().contains(q) { + out.push(group_row(l, sg, depth, g)); + all(l, sg, depth + 1, out); + continue; + } + let mut inner = vec![]; + rec(l, sg, depth + 1, q, &mut inner); + if !inner.is_empty() { + out.push(group_row(l, sg, depth, g)); + out.extend(inner); + } + } + Node::Prop(p) if prop_visible(p, l) && p.name.to_lowercase().contains(q) => { + out.push(Row { layer: l.id, depth, kind: RowKind::Prop { uid: p.uid } }); + } + _ => {} + } + } + } + let mut out = vec![]; + rec(l, &l.props, 1, q, &mut out); + out +} + +/// The property each reveal key shows (P, S, R, T, A, F, L; `animated` = keyframed or with an +/// expression): (group searched, property match ids). +fn reveal_targets(kind: &str) -> Option<(&'static str, &'static [&'static str])> { + Some(match kind { + "position" => ("transform", &["position", "positionX", "positionY", "positionZ"]), + "scale" => ("transform", &["scale"]), + "rotation" => ("transform", &["rotation", "rotationX", "rotationY", "orientation"]), + "opacity" => ("transform", &["opacity"]), + "anchor" => ("transform", &["anchor"]), + "feather" => ("masks", &["feather"]), + "levels" => ("audio", &["levels"]), + "animated" => ("", &[]), + _ => return None, + }) +} + fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { let tl = &app.ui.timeline; - let search = tl.search.to_lowercase(); + let search = tl.search.trim().to_lowercase(); let mut rows = Vec::new(); for l in &comp.layers { if comp.hide_shy && l.switches.shy { continue; } - if !search.is_empty() && !l.name.to_lowercase().contains(&search) { + if !search.is_empty() { + // Layers whose name or properties match; matching properties are revealed. + let matches = search_rows(l, &search); + if matches.is_empty() && !display_name(app, l).to_lowercase().contains(&search) { + continue; + } + rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); + rows.extend(matches); continue; } rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); if !tl.open_layers.contains(&l.id.0) { continue; } - if let Some(kind) = tl.reveal.first() { - let (group, props): (&str, Vec<&str>) = match kind.as_str() { - "position" => ("transform", vec!["position", "positionX", "positionY", "positionZ"]), - "scale" => ("transform", vec!["scale"]), - "rotation" => ("transform", vec!["rotation", "rotationX", "rotationY", "orientation"]), - "opacity" => ("transform", vec!["opacity"]), - "anchor" => ("transform", vec!["anchor"]), - "feather" => ("masks", vec!["feather"]), - "levels" => ("audio", vec!["levels"]), - _ => ("", vec![]), - }; - match kind.as_str() { - "waveform" => { - if let Some(item) = super::waveform::audio_item(&app.session.project, l) { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); - } - } - "effects" => { - if let Some(fx) = l.effects() { - for g in fx.groups() { - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { - uid: g.uid, - name: g.name.clone(), - open: tl.open_groups.contains(&g.uid), - has_children: true, - fx: Some(g.enabled), - eye: None, - }, - }); - if tl.open_groups.contains(&g.uid) { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - } - } - "masks" => { - if let Some(m) = l.masks() { - for g in m.groups() { - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { - uid: g.uid, - name: g.name.clone(), - open: tl.open_groups.contains(&g.uid), - has_children: true, - fx: None, - eye: None, - }, - }); - if tl.open_groups.contains(&g.uid) { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - } - } - "props" => { - // Animation ▸ Reveal Properties…: the engine picked the uids. - fn groups_in<'a>(g: &'a PropGroup, set: &std::collections::BTreeSet, out: &mut Vec<&'a PropGroup>) { - for sg in g.groups() { - if set.contains(&sg.uid) { - out.push(sg); - } else { - groups_in(sg, set, out); - } - } - } - let mut groups = vec![]; - groups_in(&l.props, &tl.reveal_props, &mut groups); - for g in groups { - let open = tl.open_groups.contains(&g.uid); - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open, has_children: !g.children.is_empty(), fx: None, eye: None }, - }); - if open { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - let mut found = vec![]; - collect_props(&l.props, &mut found, &|p| prop_visible(p, l) && tl.reveal_props.contains(&p.uid)); - for uid in found { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } }); - } - } - _ => { - let mut found = vec![]; - let root = if group.is_empty() { - Some(&l.props) - } else if group == "masks" { - l.masks() - } else { - l.props.sub(group) - }; - if let Some(g) = root { - collect_props(g, &mut found, &|p| prop_visible(p, l) && reveal_matches(p, &props, kind)); - } - for uid in found { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } }); - } - } - } + if !tl.reveal.is_empty() { + reveal_rows(app, l, &mut rows); continue; } for c in &l.props.children { @@ -388,6 +544,95 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { rows } +/// Rows of a twirled-open layer under the reveal shortcuts (one, or several added with Shift): +/// masks, effects, the revealed properties in property-tree order, the Reveal Properties +/// selection, then the waveform. +fn reveal_rows(app: &EffectcraftApp, l: &Layer, rows: &mut Vec) { + let tl = &app.ui.timeline; + let has = |k: &str| tl.reveal.iter().any(|r| r == k); + let group_rows = |rows: &mut Vec, g: &PropGroup, fx: bool| { + for g in g.groups() { + rows.push(Row { + layer: l.id, + depth: 1, + kind: RowKind::Group { + uid: g.uid, + name: g.name.clone(), + open: tl.open_groups.contains(&g.uid), + has_children: true, + fx: fx.then_some(g.enabled), + eye: None, + }, + }); + if tl.open_groups.contains(&g.uid) { + push_group(rows, l, g, 2, &tl.open_groups); + } + } + }; + if has("masks") + && let Some(m) = l.masks() + { + group_rows(rows, m, false); + } + if has("effects") + && let Some(fx) = l.effects() + { + group_rows(rows, fx, true); + } + // P/S/R/T/A/F/L/animated: matching properties, in tree order, each once. + let mut wanted = std::collections::BTreeSet::new(); + for kind in &tl.reveal { + let Some((group, ids)) = reveal_targets(kind) else { continue }; + let root = match group { + "" => Some(&l.props), + "masks" => l.masks(), + g => l.props.sub(g), + }; + if let Some(g) = root { + let mut found = vec![]; + collect_props(g, &mut found, &|p| prop_visible(p, l) && reveal_matches(p, ids, kind)); + wanted.extend(found); + } + } + if !wanted.is_empty() { + let mut found = vec![]; + collect_props(&l.props, &mut found, &|p| wanted.contains(&p.uid)); + rows.extend(found.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + } + if has("props") { + // Animation ▸ Reveal Properties…: the engine picked the uids. + fn groups_in<'a>(g: &'a PropGroup, set: &std::collections::BTreeSet, out: &mut Vec<&'a PropGroup>) { + for sg in g.groups() { + if set.contains(&sg.uid) { + out.push(sg); + } else { + groups_in(sg, set, out); + } + } + } + let mut groups = vec![]; + groups_in(&l.props, &tl.reveal_props, &mut groups); + for g in groups { + let open = tl.open_groups.contains(&g.uid); + rows.push(Row { + layer: l.id, + depth: 1, + kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open, has_children: !g.children.is_empty(), fx: None, eye: None }, + }); + if open { + push_group(rows, l, g, 2, &tl.open_groups); + } + } + let mut found = vec![]; + collect_props(&l.props, &mut found, &|p| prop_visible(p, l) && tl.reveal_props.contains(&p.uid) && !wanted.contains(&p.uid)); + rows.extend(found.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + } + if has("waveform") + && let Some(item) = super::waveform::audio_item(&app.session.project, l) + { + rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); + } +} fn collect_props(g: &PropGroup, out: &mut Vec, f: &dyn Fn(&Property) -> bool) { for c in &g.children { match c { @@ -467,9 +712,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let ruler_h = 34.0; let colhdr_h = 22.0; let footer_h = 24.0; - let fixed = AV_W + LABEL_W + NUM_W + SW * 8.0 + if app.ui.timeline.show_modes { 92.0 + 112.0 } else { 0.0 } + 116.0 + 6.0; - let left_w = (fixed + 190.0).clamp(420.0, (rect.width() * 0.62).max(420.0)); - let cw = cols(rect.min.x, left_w, app.ui.timeline.show_modes); + let vis = Vis::of(&app.ui.timeline); + let left_w = (vis.fixed() + 190.0).clamp(420.0, (rect.width() * 0.72).max(420.0)); + let cw = cols(rect.min.x, left_w, vis); let graph_x0 = rect.min.x + left_w + 1.0; let graph_x1 = rect.max.x - 10.0; ctx.data_mut(|d| d.insert_temp(egui::Id::new("timeline-graph-area"), (graph_x0, graph_x1 - graph_x0))); @@ -487,6 +732,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { p.rect_filled(Rect::from_min_max(pos2(graph_x0, top), rect.max), 0.0, t.tl_bg); p.line_segment([pos2(graph_x0 - 1.0, top), pos2(graph_x0 - 1.0, rect.max.y)], Stroke::new(1.0, t.app_bg)); + let mut actions: Vec<(String, serde_json::Value)> = Vec::new(); // ---- header (left): time display, search, switches. let tc = crate::panels::timecode(&app.session, &comp, time); let tc_rect = Rect::from_min_size(pos2(rect.min.x + 12.0, top + 6.0), vec2(150.0, 24.0)); @@ -636,36 +882,118 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if (rresp.dragged() || rresp.clicked()) && let Some(pt) = rresp.interact_pointer_pos() { - let tt = Tick::from_seconds_f64(tm.t(pt.x).max(0.0)); + let mut secs = tm.t(pt.x).max(0.0); + // Shift-drag: snap to keyframes, in/out points, markers and the work area (8 px). + if ui.input(|i| i.modifiers.shift) { + let cands = snap_candidates(&comp, &build_rows(app, &comp)); + secs = snap_time(&cands, secs, 8.0 / pps.max(1e-6)); + } + let tt = Tick::from_seconds_f64(secs); app.session.set_time(tt); app.stop(); } - // ---- column headers. + // ---- column headers (right-click: show/hide columns; click the name header: Source/Layer Name). let ch = Rect::from_min_max(pos2(rect.min.x, top + header_h), pos2(rect.max.x, top + header_h + colhdr_h)); p.rect_filled(Rect::from_min_max(ch.min, pos2(graph_x0 - 1.0, ch.max.y)), 0.0, t.panel_bg); p.line_segment([pos2(rect.min.x, ch.max.y), pos2(rect.max.x, ch.max.y)], Stroke::new(1.0, t.separator)); let hy = ch.center().y; let hic = |icon: Icon, x: f32| icons::paint(&p, Rect::from_center_size(pos2(x, hy), vec2(12.0, 12.0)), icon, t.text_dim); - hic(Icon::Eye, cw.av + 10.0); - hic(Icon::Speaker, cw.av + 28.0); - hic(Icon::Solo, cw.av + 46.0); - hic(Icon::Lock, cw.av + 64.0); - icons::paint(&p, Rect::from_center_size(pos2(cw.label + 10.0, hy), vec2(10.0, 10.0)), Icon::Keyframe, t.text_dim); - p.text(pos2(cw.num + 6.0, hy), Align2::LEFT_CENTER, "#", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.name + 22.0, hy), Align2::LEFT_CENTER, "Layer Name", Tokens::ui(11.0), t.text_dim); - for (i, icon) in - [Icon::Shy, Icon::Collapse, Icon::Quality, Icon::Fx, Icon::FrameBlend, Icon::MotionBlur, Icon::Adjustment, Icon::Cube].into_iter().enumerate() + let htext = |x: f32, s: &str| p.text(pos2(x, hy), Align2::LEFT_CENTER, s, Tokens::ui(11.0), t.text_dim); + if vis.av { + hic(Icon::Eye, cw.av + 10.0); + hic(Icon::Speaker, cw.av + 28.0); + hic(Icon::Solo, cw.av + 46.0); + hic(Icon::Lock, cw.av + 64.0); + } + if vis.keys { + htext(cw.keys + 6.0, "Keys"); + } + if vis.label { + icons::paint(&p, Rect::from_center_size(pos2(cw.label + 10.0, hy), vec2(10.0, 10.0)), Icon::Keyframe, t.text_dim); + } + if vis.num { + htext(cw.num + 6.0, "#"); + } + let name_hdr = if app.ui.timeline.source_name { "Source Name" } else { "Layer Name" }; + htext(cw.name + 22.0, name_hdr); + if vis.comment { + htext(cw.comment + 6.0, "Comment"); + } + if vis.switches { + for (i, icon) in + [Icon::Shy, Icon::Collapse, Icon::Quality, Icon::Fx, Icon::FrameBlend, Icon::MotionBlur, Icon::Adjustment, Icon::Cube].into_iter().enumerate() + { + hic(icon, cw.switches + SW * i as f32 + SW / 2.0); + } + } + if vis.modes { + htext(cw.mode + 4.0, "Mode"); + htext(cw.mode + 74.0, "T"); + htext(cw.trkmat + 4.0, "Track Matte"); + } + if vis.parent { + icons::paint(&p, Rect::from_center_size(pos2(cw.parent + 10.0, hy), vec2(12.0, 12.0)), Icon::PickWhip, t.text_dim); + htext(cw.parent + 20.0, "Parent & Link"); + } + for (on, x, s) in [(vis.in_, cw.in_, "In"), (vis.out, cw.out, "Out"), (vis.duration, cw.duration, "Duration"), (vis.stretch, cw.stretch, "Stretch")] { + if on { + htext(x + 6.0, s); + } + } + let hdr_rect = Rect::from_min_max(ch.min, pos2(graph_x0 - 1.0, ch.max.y)); + let hresp = ui.interact(hdr_rect, egui::Id::new("tl-colhdr"), Sense::click()); + app.auto.add("timeline.header", hdr_rect, "Columns (right-click)"); + for (id, label) in COLUMNS { + if let Some(x0) = match id { + "av" => vis.av.then_some((cw.av, AV_W)), + "keys" => vis.keys.then_some((cw.keys, KEYS_W)), + "label" => vis.label.then_some((cw.label, LABEL_W)), + "num" => vis.num.then_some((cw.num, NUM_W)), + "name" => Some((cw.name, cw.name_end - cw.name)), + "comment" => vis.comment.then_some((cw.comment, COMMENT_W)), + "switches" => vis.switches.then_some((cw.switches, SW * 8.0 + 6.0)), + "modes" => vis.modes.then_some((cw.mode, MODES_W)), + "parent" => vis.parent.then_some((cw.parent, PARENT_W)), + "in" => vis.in_.then_some((cw.in_, TIME_W)), + "out" => vis.out.then_some((cw.out, TIME_W)), + "duration" => vis.duration.then_some((cw.duration, TIME_W)), + _ => vis.stretch.then_some((cw.stretch, STRETCH_W)), + } { + app.auto.add(&format!("timeline.header.{id}"), Rect::from_min_size(pos2(x0.0, ch.min.y), vec2(x0.1, colhdr_h)), label); + } + } + if hresp.clicked() + && let Some(pt) = hresp.interact_pointer_pos() + && column_at(&cw, pt.x) == Some("name") { - hic(icon, cw.switches + SW * i as f32 + SW / 2.0); + app.ui.timeline.source_name = !app.ui.timeline.source_name; } - if app.ui.timeline.show_modes { - p.text(pos2(cw.mode + 4.0, hy), Align2::LEFT_CENTER, "Mode", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.mode + 74.0, hy), Align2::LEFT_CENTER, "T", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.trkmat + 4.0, hy), Align2::LEFT_CENTER, "Track Matte", Tokens::ui(11.0), t.text_dim); + if hresp.secondary_clicked() { + let under = hresp.interact_pointer_pos().and_then(|pt| column_at(&cw, pt.x)); + ctx.data_mut(|d| d.insert_temp(egui::Id::new("tl-colhdr-under"), under.unwrap_or(""))); } - icons::paint(&p, Rect::from_center_size(pos2(cw.parent + 10.0, hy), vec2(12.0, 12.0)), Icon::PickWhip, t.text_dim); - p.text(pos2(cw.parent + 20.0, hy), Align2::LEFT_CENTER, "Parent & Link", Tokens::ui(11.0), t.text_dim); + hresp.context_menu(|ui| { + let under: &str = ctx.data(|d| d.get_temp(egui::Id::new("tl-colhdr-under"))).unwrap_or(""); + if !under.is_empty() && under != "name" && ui.button("Hide This").clicked() { + actions.push(("timeline.column".into(), json!({"column": under, "visible": false}))); + ui.close(); + } + ui.menu_button("Columns", |ui| { + for (id, label) in COLUMNS { + let label = if id == "name" { if app.ui.timeline.source_name { "Source Name" } else { "Layer Name" } } else { label }; + let on = column_visible(&app.ui.timeline, id); + if ui.add_enabled(id != "name", egui::Button::new(label).selected(on)).clicked() { + actions.push(("timeline.column".into(), json!({"column": id, "visible": !on}))); + ui.close(); + } + } + }); + if ui.button(if app.ui.timeline.source_name { "Show Layer Name" } else { "Show Source Name" }).clicked() { + actions.push(("timeline.sourceName".into(), json!({}))); + ui.close(); + } + }); let _ = cw.end; // ---- rows. @@ -693,7 +1021,6 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let mut y = rows_rect.min.y - app.ui.timeline.scroll_y; let selected = app.session.state.selected_layers.clone(); let sel_keys = app.session.state.selected_keys.clone(); - let mut actions: Vec<(String, serde_json::Value)> = Vec::new(); let mut ui_actions: Vec = Vec::new(); let idx_of = |id: LayerId| comp.index_of(id).unwrap_or(0); let graph_on = app.ui.timeline.graph_editor; @@ -743,7 +1070,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ]; for (i, (icon, on, name, applicable)) in av_items.into_iter().enumerate() { let br = Rect::from_center_size(pos2(cw.av + 10.0 + 18.0 * i as f32, cy), vec2(17.0, 17.0)); - if !applicable { + if !applicable || !vis.av { continue; } lp.rect_stroke(br.shrink(1.5), 2.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); @@ -754,19 +1081,24 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } // Label swatch. - let lr = Rect::from_center_size(pos2(cw.label + 10.0, cy), vec2(12.0, 12.0)); - lp.rect_filled(lr, 2.0, t.label(layer.label)); - let lresp = ui.interact(lr, egui::Id::new(("label", layer.id.0)), Sense::click()); - lresp.context_menu(|ui| { - for lab in effectcraft_engine::color::Label::ALL { - let name = if lab == effectcraft_engine::color::Label::None { lab.name().to_string() } else { app.session.prefs.label_name(lab) }; - if ui.button(name).clicked() { - actions.push(("edit.label".into(), json!({"layers": [layer.id.0], "label": lab.name()}))); - ui.close(); + if vis.label { + let lr = Rect::from_center_size(pos2(cw.label + 10.0, cy), vec2(12.0, 12.0)); + lp.rect_filled(lr, 2.0, t.label(layer.label)); + let lresp = ui.interact(lr, egui::Id::new(("label", layer.id.0)), Sense::click()); + lresp.context_menu(|ui| { + for lab in effectcraft_engine::color::Label::ALL { + let name = if lab == effectcraft_engine::color::Label::None { lab.name().to_string() } else { app.session.prefs.label_name(lab) }; + if ui.button(name).clicked() { + actions.push(("edit.label".into(), json!({"layers": [layer.id.0], "label": lab.name()}))); + ui.close(); + } } - } - }); - lp.text(pos2(cw.num + 13.0, cy), Align2::CENTER_CENTER, format!("{}", idx_of(layer.id)), Tokens::ui(11.5), t.text_dim); + }); + } + if vis.num { + lp.text(pos2(cw.num + 13.0, cy), Align2::CENTER_CENTER, format!("{}", idx_of(layer.id)), Tokens::ui(11.5), t.text_dim); + } + layer_cells(app, ui, &lp, &cw, &comp, layer, r, cy, &mut actions); // Twirl + icon + name. let tw = Rect::from_center_size(pos2(cw.name + 8.0, cy), vec2(14.0, 14.0)); let open = app.ui.timeline.open_layers.contains(&layer.id.0); @@ -775,7 +1107,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } app.auto.add(&format!("timeline.layer.{}.twirl", layer.id.0), tw, "twirl"); icons::paint(&lp, Rect::from_center_size(pos2(cw.name + 24.0, cy), vec2(13.0, 13.0)), layer_icon(layer, &app.session.project), t.text_dim); - let name_rect = Rect::from_min_max(pos2(cw.name + 34.0, r.min.y), pos2(cw.switches - 4.0, r.max.y)); + let name_rect = Rect::from_min_max(pos2(cw.name + 34.0, r.min.y), pos2(cw.name_end - 4.0, r.max.y)); let rename_id = egui::Id::new("tl-rename"); let renaming: Option<(u64, String)> = ctx.data(|d| d.get_temp(rename_id)); if let Some((rl, mut buf)) = renaming.filter(|(rl, _)| *rl == layer.id.0) { @@ -793,11 +1125,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } else { let name_col = if is_sel { Color32::WHITE } else { t.text }; let lpn = lp.with_clip_rect(name_rect.intersect(lp.clip_rect())); - lpn.text(pos2(name_rect.min.x, cy), Align2::LEFT_CENTER, &layer.name, Tokens::ui(12.0), name_col); + lpn.text(pos2(name_rect.min.x, cy), Align2::LEFT_CENTER, display_name(app, layer), Tokens::ui(12.0), name_col); } // Row click → select; double-click → rename; drag → reorder (later). + let row_x0 = if vis.num { cw.num } else { cw.name + 16.0 }; let row_resp = - ui.interact(Rect::from_min_max(pos2(cw.num, r.min.y), pos2(cw.switches, r.max.y)), egui::Id::new(("row", layer.id.0)), Sense::click()); + ui.interact(Rect::from_min_max(pos2(row_x0, r.min.y), pos2(cw.name_end, r.max.y)), egui::Id::new(("row", layer.id.0)), Sense::click()); app.auto.add(&format!("timeline.layer.{}.row", layer.id.0), left, &layer.name); if row_resp.clicked() { let m = ui.input(|i| i.modifiers); @@ -833,7 +1166,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { (Icon::Cube, layer.is_3d(), "threeD", layer.source.is_av() || matches!(layer.source, LayerSource::Null)), ]; for (i, (icon, on, name, applicable)) in sws.into_iter().enumerate() { - if !applicable { + if !applicable || !vis.switches { continue; } let br = Rect::from_center_size(pos2(cw.switches + SW * i as f32 + SW / 2.0, cy), vec2(16.0, 16.0)); @@ -904,28 +1237,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } // Parent. - let pw_rect = Rect::from_center_size(pos2(cw.parent + 9.0, cy), vec2(15.0, 15.0)); - let pw = ui.interact(pw_rect, egui::Id::new(("parent-whip", layer.id.0)), Sense::drag()).on_hover_text("Parent pick whip: drag onto a layer"); - icons::paint(&lp, pw_rect.shrink(1.0), Icon::PickWhip, if pw.hovered() || pw.dragged() { t.text } else { t.text_dim }); - app.auto.add(&format!("timeline.layer.{}.pickWhip", layer.id.0), pw_rect, "Parent pick whip"); - if pw.drag_started() { - let pw_state: PickWhip = (0, layer.id.0, 0, pw_rect.center()); - ctx.data_mut(|d| d.insert_temp(pick_whip_id(), pw_state)); - } - let pr_rect = Rect::from_min_size(pos2(cw.parent + 20.0, cy - 9.0), vec2(94.0, 18.0)); - let plabel = layer.parent.and_then(|p| comp.layer(p)).map(|l| format!("{}. {}", idx_of(l.id), l.name)).unwrap_or_else(|| "None".into()); - let presp = widgets::dropdown(ui, pr_rect, &plabel, &t, egui::Id::new(("parent", layer.id.0))); - app.auto.add(&format!("timeline.layer.{}.parent", layer.id.0), pr_rect, "Parent"); - let pop = egui::Id::new(("par-pop", layer.id.0)); - if presp.clicked() { - widgets::open_popup(ui, pop); - } - let cands: Vec<&Layer> = comp.layers.iter().filter(|l| l.id != layer.id).collect(); - let mut popts = vec!["None".to_string(), "-".to_string()]; - popts.extend(cands.iter().map(|l| format!("{}. {}", idx_of(l.id), l.name))); - if let Some(i) = widgets::popup_menu(ui, pop, pr_rect.left_bottom(), &popts, None) { - let par = if i == 0 { serde_json::Value::Null } else { json!(cands[i - 2].id.0) }; - actions.push(("layer.setParent".into(), json!({"layers": [layer.id.0], "parent": par}))); + if vis.parent { + let pw_rect = Rect::from_center_size(pos2(cw.parent + 9.0, cy), vec2(15.0, 15.0)); + let pw = + ui.interact(pw_rect, egui::Id::new(("parent-whip", layer.id.0)), Sense::drag()).on_hover_text("Parent pick whip: drag onto a layer"); + icons::paint(&lp, pw_rect.shrink(1.0), Icon::PickWhip, if pw.hovered() || pw.dragged() { t.text } else { t.text_dim }); + app.auto.add(&format!("timeline.layer.{}.pickWhip", layer.id.0), pw_rect, "Parent pick whip"); + if pw.drag_started() { + let pw_state: PickWhip = (0, layer.id.0, 0, pw_rect.center()); + ctx.data_mut(|d| d.insert_temp(pick_whip_id(), pw_state)); + } + let pr_rect = Rect::from_min_size(pos2(cw.parent + 20.0, cy - 9.0), vec2(94.0, 18.0)); + let plabel = layer.parent.and_then(|p| comp.layer(p)).map(|l| format!("{}. {}", idx_of(l.id), l.name)).unwrap_or_else(|| "None".into()); + let presp = widgets::dropdown(ui, pr_rect, &plabel, &t, egui::Id::new(("parent", layer.id.0))); + app.auto.add(&format!("timeline.layer.{}.parent", layer.id.0), pr_rect, "Parent"); + let pop = egui::Id::new(("par-pop", layer.id.0)); + if presp.clicked() { + widgets::open_popup(ui, pop); + } + let cands: Vec<&Layer> = comp.layers.iter().filter(|l| l.id != layer.id).collect(); + let mut popts = vec!["None".to_string(), "-".to_string()]; + popts.extend(cands.iter().map(|l| format!("{}. {}", idx_of(l.id), l.name))); + if let Some(i) = widgets::popup_menu(ui, pop, pr_rect.left_bottom(), &popts, None) { + let par = if i == 0 { serde_json::Value::Null } else { json!(cands[i - 2].id.0) }; + actions.push(("layer.setParent".into(), json!({"layers": [layer.id.0], "parent": par}))); + } } // Layer bar. ui.set_clip_rect(right_clip); @@ -1190,10 +1526,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, t.tl_bg); let indent = cw.name + 6.0 + 14.0 * row.depth as f32; // Keyframe navigator (in the A/V column, like AE). - if prop.is_animated() { + // (in the Keys column when it is shown). + if prop.is_animated() && (vis.keys || vis.av) { let lt = layer.layer_time(time); let at_key = effectcraft_engine::keyframe::key_at(&prop.keys, lt).is_some(); - let nav_x = cw.av + 18.0; + let nav_x = if vis.keys { cw.keys + 12.0 } else { cw.av + 18.0 }; let prev = Rect::from_center_size(pos2(nav_x, cy), vec2(12.0, 14.0)); let mid = Rect::from_center_size(pos2(nav_x + 16.0, cy), vec2(14.0, 14.0)); let next = Rect::from_center_size(pos2(nav_x + 32.0, cy), vec2(12.0, 14.0)); @@ -1556,6 +1893,101 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// The optional per-layer cells: Comment (double-click to edit), In / Out / Duration (drag to +/// change by frames), Stretch (click: Time Stretch dialog). +#[allow(clippy::too_many_arguments)] +fn layer_cells( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + lp: &egui::Painter, + cw: &Cols, + comp: &Comp, + layer: &Layer, + r: Rect, + cy: f32, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let lid = layer.id.0; + if cw.vis.comment { + let cr = Rect::from_min_max(pos2(cw.comment + 4.0, r.min.y + 1.0), pos2(cw.comment + COMMENT_W - 4.0, r.max.y - 1.0)); + let edit_id = egui::Id::new("tl-comment-edit"); + let editing: Option<(u64, String)> = ctx.data(|d| d.get_temp(edit_id)); + app.auto.add(&format!("timeline.layer.{lid}.comment"), cr, &layer.comment); + if let Some((el, mut buf)) = editing.filter(|(el, _)| *el == lid) { + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(cr)); + let er = child.add(egui::TextEdit::singleline(&mut buf).font(Tokens::ui(11.5)).desired_width(cr.width())); + er.request_focus(); + if er.lost_focus() { + if !ui.input(|i| i.key_pressed(egui::Key::Escape)) && buf != layer.comment { + actions.push(("layer.setComment".into(), json!({"layers": [el], "comment": buf}))); + } + ctx.data_mut(|d| d.remove::<(u64, String)>(edit_id)); + } else { + ctx.data_mut(|d| d.insert_temp(edit_id, (el, buf))); + } + } else { + let resp = ui.interact(cr, egui::Id::new(("tl-comment", lid)), Sense::click()).on_hover_text("Double-click to edit the comment"); + lp.with_clip_rect(cr.intersect(lp.clip_rect())).text(pos2(cr.min.x + 2.0, cy), Align2::LEFT_CENTER, &layer.comment, Tokens::ui(11.5), t.text_dim); + if resp.double_clicked() { + ctx.data_mut(|d| d.insert_temp(edit_id, (lid, layer.comment.clone()))); + } + } + } + let fd = comp.frame_duration().seconds(); + let cells = [ + (cw.vis.in_, cw.in_, "in", layer.in_point.seconds()), + (cw.vis.out, cw.out, "out", layer.out_point.seconds()), + (cw.vis.duration, cw.duration, "duration", (layer.out_point - layer.in_point).seconds()), + ]; + for (on, x, kind, secs) in cells { + if !on { + continue; + } + let cr = Rect::from_min_max(pos2(x + 2.0, r.min.y + 1.0), pos2(x + TIME_W - 2.0, r.max.y - 1.0)); + let tc = crate::panels::timecode(&app.session, comp, Tick::from_seconds_f64(secs) - if kind == "duration" { comp.display_start } else { Tick(0) }); + let resp = ui.interact(cr, egui::Id::new(("tl-cell", kind, lid)), Sense::drag()).on_hover_text("Drag to change"); + lp.text(pos2(cr.min.x + 4.0, cy), Align2::LEFT_CENTER, &tc, Tokens::mono(11.0), if resp.hovered() || resp.dragged() { t.hot_text } else { t.timecode }); + app.auto.add(&format!("timeline.layer.{lid}.{kind}"), cr, &tc); + if resp.hovered() || resp.dragged() { + ctx.set_cursor_icon(egui::CursorIcon::ResizeHorizontal); + } + let acc_id = egui::Id::new(("tl-cell-acc", kind, lid)); + if resp.dragged() { + let mut acc: f32 = ctx.data(|d| d.get_temp(acc_id).unwrap_or(0.0)); + acc += resp.drag_delta().x; + let frames = (acc / 4.0).trunc() as i64; + if frames != 0 { + acc -= frames as f32 * 4.0; + let d = frames as f64 * fd; + let merge = format!("cell-{kind}-{lid}"); + let params = match kind { + "in" => json!({"layers": [lid], "in": (layer.in_point.seconds() + d).max(0.0), "merge": merge}), + _ => json!({"layers": [lid], "out": (layer.out_point.seconds() + d).max(layer.in_point.seconds() + fd), "merge": merge}), + }; + actions.push(("layer.timing".into(), params)); + } + ctx.data_mut(|d| d.insert_temp(acc_id, acc)); + } + if resp.drag_stopped() { + ctx.data_mut(|d| d.remove::(acc_id)); + actions.push(("__endMerge".into(), json!({}))); + } + } + if cw.vis.stretch { + let cr = Rect::from_min_max(pos2(cw.stretch + 2.0, r.min.y + 1.0), pos2(cw.stretch + STRETCH_W - 2.0, r.max.y - 1.0)); + let label = format!("{:.1}%", layer.stretch * 100.0); + let resp = ui.interact(cr, egui::Id::new(("tl-stretch", lid)), Sense::click()).on_hover_text("Time Stretch…"); + lp.text(pos2(cr.min.x + 4.0, cy), Align2::LEFT_CENTER, &label, Tokens::ui(11.5), if resp.hovered() { t.hot_text } else { t.text }); + app.auto.add(&format!("timeline.layer.{lid}.stretch"), cr, &label); + if resp.clicked() { + actions.push(("layer.select".into(), json!({"layers": [lid]}))); + actions.push(("layer.timeStretch".into(), json!({}))); + } + } +} + enum UiAct { ToggleLayer(u64, bool), ToggleGroup(u64), @@ -1856,3 +2288,136 @@ fn value_editor( } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A comp with two solids; "Box" has an animated Position and a Gaussian Blur. + fn app() -> EffectcraftApp { + let mut s = effectcraft_engine::Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 180, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": 320, "height": 180})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 40, "height": 40})).unwrap(); + let comp = s.active_comp().unwrap(); + let bx = comp.layers.iter().find(|l| l.name == "Box").unwrap().id.0; + s.execute("layer.select", json!({"layers": [bx]})).unwrap(); + s.execute("effect.apply", json!({"effect": "ec.blur.gaussian"})).unwrap(); + let pos = s.active_comp().unwrap().layer(LayerId(bx)).unwrap().transform().unwrap().get("position").unwrap().uid; + s.execute("prop.toggleAnimation", json!({"layer": bx, "prop": pos})).unwrap(); + EffectcraftApp::new(s) + } + + fn labels(app: &EffectcraftApp) -> Vec { + let comp = app.session.active_comp().unwrap().clone(); + build_rows(app, &comp) + .into_iter() + .map(|r| { + let l = comp.layer(r.layer).unwrap(); + match r.kind { + RowKind::Layer => l.name.clone(), + RowKind::Group { name, .. } => format!("{}>{name}", " ".repeat(r.depth)), + RowKind::Prop { uid } => format!("{}{}", " ".repeat(r.depth), l.props.find(uid).unwrap().name), + _ => "?".into(), + } + }) + .collect() + } + + #[test] + fn search_filters_layers_and_reveals_matching_properties() { + let mut app = app(); + app.ui.timeline.search = "opac".into(); + let rows = labels(&app); + // Both layers have Transform > Opacity; nothing else shows. + assert_eq!(rows.iter().filter(|r| r.trim() == "Opacity").count(), 2, "{rows:?}"); + assert!(rows.iter().any(|r| r.trim() == ">Transform"), "{rows:?}"); + assert!(!rows.iter().any(|r| r.trim() == "Position"), "{rows:?}"); + // A layer name match shows the layer alone; a group match brings its contents. + app.ui.timeline.search = "plate".into(); + assert_eq!(labels(&app), vec!["Plate".to_string()]); + app.ui.timeline.search = "GAUSSIAN".into(); + let rows = labels(&app); + assert_eq!(rows[0], "Box"); + assert!(rows.iter().any(|r| r.trim() == ">Effects") && rows.iter().any(|r| r.trim() == "Blurriness"), "{rows:?}"); + assert!(!rows.contains(&"Plate".to_string())); + // Nothing matches → no rows. + app.ui.timeline.search = "zzz".into(); + assert!(labels(&app).is_empty()); + // Through the command. + let ctx = egui::Context::default(); + crate::menus::invoke(&mut app, &ctx, "timeline.search", json!({"query": "scale"})).unwrap(); + assert!(labels(&app).iter().any(|r| r.trim() == "Scale")); + } + + #[test] + fn columns_toggle_and_layout() { + let mut app = app(); + let tl = &mut app.ui.timeline; + assert!(column_visible(tl, "switches") && !column_visible(tl, "comment") && !column_visible(tl, "modes")); + set_column(tl, "comment", true).unwrap(); + set_column(tl, "stretch", true).unwrap(); + set_column(tl, "keys", false).unwrap(); + set_column(tl, "label", false).unwrap(); + set_column(tl, "modes", true).unwrap(); + assert!(tl.show_modes && column_visible(tl, "comment") && !column_visible(tl, "label")); + assert!(set_column(tl, "name", false).is_err()); + assert!(set_column(tl, "bogus", true).is_err()); + let vis = Vis::of(tl); + let cw = cols(0.0, 1400.0, vis); + // Hidden columns take no room; visible ones are in After Effects' order. + assert_eq!(cw.keys, cw.label); + assert_eq!(cw.label, cw.num); + assert!(cw.name < cw.comment && cw.comment < cw.switches && cw.switches < cw.mode && cw.parent < cw.stretch); + assert_eq!(column_at(&cw, cw.comment + 5.0), Some("comment")); + assert_eq!(column_at(&cw, cw.name + 5.0), Some("name")); + assert_eq!(column_at(&cw, cw.stretch + 5.0), Some("stretch")); + // Through the command (as agents do). + let ctx = egui::Context::default(); + let r = crate::menus::invoke(&mut app, &ctx, "timeline.column", json!({"column": "duration"})).unwrap(); + assert!(r["columns"].as_array().unwrap().iter().any(|c| c == "duration")); + crate::menus::invoke(&mut app, &ctx, "timeline.column", json!({"column": "duration", "visible": false})).unwrap(); + assert!(!column_visible(&app.ui.timeline, "duration")); + crate::menus::invoke(&mut app, &ctx, "timeline.sourceName", json!({})).unwrap(); + assert!(app.ui.timeline.source_name); + } + + #[test] + fn reveal_keys_add_with_shift_and_uu_reveals_modified() { + let mut app = app(); + let ctx = egui::Context::default(); + crate::menus::invoke(&mut app, &ctx, "timeline.reveal.opacity", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity"]); + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.scale", json!({})).unwrap(); + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.effects", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity", "scale", "effects"]); + let rows = labels(&app); + assert!(rows.iter().any(|r| r.trim() == "Scale") && rows.iter().any(|r| r.trim() == "Opacity"), "{rows:?}"); + assert!(rows.iter().any(|r| r.trim() == ">Gaussian Blur"), "{rows:?}"); + // Tree order: Scale before Opacity. + let si = rows.iter().position(|r| r.trim() == "Scale").unwrap(); + let oi = rows.iter().position(|r| r.trim() == "Opacity").unwrap(); + assert!(si < oi); + // Shift+S again removes Scale. + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.scale", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity", "effects"]); + // U: keyframed properties (Position); UU (twice quickly): every modified property. + crate::menus::invoke(&mut app, &ctx, "anim.reveal", json!({"kind": "keyframes"})).unwrap(); + let n_keys = app.ui.timeline.reveal_props.len(); + assert_eq!(app.ui.timeline.reveal, vec!["props"]); + assert!(n_keys >= 1); + crate::menus::invoke(&mut app, &ctx, "anim.reveal", json!({"kind": "keyframes"})).unwrap(); + assert!(app.ui.timeline.reveal_props.len() > n_keys, "UU reveals modified properties too"); + } + + #[test] + fn cti_snaps_to_the_nearest_candidate() { + assert_eq!(snap_time(&[0.0, 1.0, 2.0], 1.04, 0.1), 1.0); + assert_eq!(snap_time(&[0.0, 1.0, 2.0], 1.5, 0.1), 1.5); + assert_eq!(snap_time(&[1.0, 1.1], 1.06, 0.1), 1.1); + let app = app(); + let comp = app.session.active_comp().unwrap().clone(); + let c = snap_candidates(&comp, &build_rows(&app, &comp)); + assert!(c.contains(&comp.duration.seconds()) && c.contains(&0.0)); + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d13501032..af59bb6b7 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -360,6 +360,18 @@ pub struct TimelineState { /// Properties whose inline expression editor is collapsed. #[serde(default)] pub expr_closed: BTreeSet, + /// Visible optional columns (column header right-click ▸ Columns): `av`, `keys`, `label`, + /// `num`, `comment`, `switches`, `parent`, `in`, `out`, `duration`, `stretch`. The name + /// column is always shown; Modes follows `show_modes` (F4). + #[serde(default = "default_tl_columns")] + pub columns: BTreeSet, + /// The name column shows Source Name instead of Layer Name (click its header). + #[serde(default)] + pub source_name: bool, +} + +pub fn default_tl_columns() -> BTreeSet { + ["av", "label", "num", "switches", "parent"].map(String::from).into_iter().collect() } fn value_graph() -> String { @@ -391,6 +403,8 @@ impl Default for TimelineState { graph_reference: false, graph_transform_box: true, expr_closed: BTreeSet::new(), + columns: default_tl_columns(), + source_name: false, } } } From a86b02901e7d3c9134f41968bc396bcd7ecbab71 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:26:14 -0400 Subject: [PATCH 08/14] UI-7: Home screen with recent projects, thumbnails and community links The Home screen (panels/home.rs) gets New Project / Open Project / New Composition / Open Demo Project / Import, the recent projects from Settings (File > Open Recent) with thumbnails, folder and modified date (click opens, Clear list), and the ArtCraft community links (Discord, getartcraft.com, the EffectCraft page, GitHub, sibling apps) where After Effects has Learn / What's New. A 96x54 thumbnail is stored in the config store when a project is opened or saved and the viewer shows it. Automation ids: home., home.recent., home.clearRecent, home.sibling.. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/lib.rs | 7 + crates/ui-egui/src/panels/home.rs | 389 ++++++++++++++++++++++++++++++ crates/ui-egui/src/panels/misc.rs | 72 +----- crates/ui-egui/src/panels/mod.rs | 3 +- 4 files changed, 399 insertions(+), 72 deletions(-) create mode 100644 crates/ui-egui/src/panels/home.rs diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e903811de..59005cdc6 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -164,6 +164,10 @@ pub struct EffectcraftApp { /// Docked groups laid out last frame: (active panel, group rect) — `~` maximizes the one /// under the pointer. pub(crate) dock_rects: Vec<(PanelKind, egui::Rect)>, + /// Home screen: recent-project thumbnail textures by path (None = no thumbnail). + pub(crate) home_thumbs: std::collections::HashMap>, + /// The (project path, saved revision) whose thumbnail was stored last. + pub(crate) home_thumb_saved: Option<(String, u64)>, } impl EffectcraftApp { @@ -208,6 +212,8 @@ impl EffectcraftApp { applied_prefs: None, recovery: None, dock_rects: vec![], + home_thumbs: Default::default(), + home_thumb_saved: None, } .with_ui_commands() } @@ -762,6 +768,7 @@ impl EffectcraftApp { let body = egui::Rect::from_min_max(egui::pos2(full.min.x + 4.0, header.max.y + 2.0), egui::pos2(full.max.x - 4.0, full.max.y - 4.0)); self.dock_area(ui, body); panels::precomp::mini_flowchart(self, &ctx); + panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/panels/home.rs b/crates/ui-egui/src/panels/home.rs new file mode 100644 index 000000000..42ded4b74 --- /dev/null +++ b/crates/ui-egui/src/panels/home.rs @@ -0,0 +1,389 @@ +//! The Home screen (shown in the Composition panel at launch, from the header's Home button, and +//! by the Learn workspace): New Project / Open Project / New Composition / Open Demo, the recent +//! projects (File ▸ Open Recent, from Settings) with thumbnails, folders and dates, and where +//! After Effects has its Learn / What's New area, the ArtCraft community links. +//! +//! Thumbnails: when a project is opened or saved and the viewer has its frame, a 96×54 RGB +//! thumbnail is stored in the config store (`thumb-.txt`, hex) and shown here. + +use std::hash::{Hash, Hasher}; + +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde::Serialize; +use serde_json::json; + +use crate::icons::{self, Icon}; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +pub const THUMB_W: usize = 96; +pub const THUMB_H: usize = 54; + +/// One row of the recent projects list. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct RecentEntry { + pub index: usize, + pub path: String, + pub name: String, + pub folder: String, + /// Last modified (`2026-10-02 14:05`, UTC), when the file is readable. + pub modified: Option, + pub exists: bool, +} + +/// The recent projects, most recent first (Settings ▸ General ▸ recent items). +pub fn recent_entries(prefs: &effectcraft_engine::prefs::Prefs) -> Vec { + prefs + .recent_projects + .iter() + .enumerate() + .map(|(index, p)| { + let path = std::path::Path::new(p); + let meta = std::fs::metadata(path).ok(); + let modified = meta + .as_ref() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| format_utc(d.as_secs() as i64)); + RecentEntry { + index, + path: p.clone(), + name: path.file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| p.clone()), + folder: path.parent().map(|d| d.to_string_lossy().to_string()).unwrap_or_default(), + modified, + exists: meta.is_some(), + } + }) + .collect() +} + +/// `YYYY-MM-DD HH:MM` (UTC) from Unix seconds. +pub fn format_utc(secs: i64) -> String { + let days = secs.div_euclid(86_400); + let rem = secs.rem_euclid(86_400); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02} {:02}:{:02}", rem / 3600, (rem % 3600) / 60) +} + +/// The config-store name of a project's thumbnail. +pub fn thumb_name(path: &str) -> String { + let mut h = std::collections::hash_map::DefaultHasher::new(); + path.hash(&mut h); + format!("thumb-{:016x}.txt", h.finish()) +} + +/// Downscale an image to a letterboxed THUMB_W×THUMB_H RGB thumbnail (hex text). +pub fn encode_thumb(img: &egui::ColorImage) -> String { + let [w, h] = img.size; + let mut out = String::with_capacity(THUMB_W * THUMB_H * 6); + let k = (THUMB_W as f32 / w.max(1) as f32).min(THUMB_H as f32 / h.max(1) as f32); + let (dw, dh) = (w as f32 * k, h as f32 * k); + let (ox, oy) = ((THUMB_W as f32 - dw) / 2.0, (THUMB_H as f32 - dh) / 2.0); + for y in 0..THUMB_H { + for x in 0..THUMB_W { + let (fx, fy) = ((x as f32 + 0.5 - ox) / k, (y as f32 + 0.5 - oy) / k); + let c = if fx >= 0.0 && fy >= 0.0 && (fx as usize) < w && (fy as usize) < h { + let p = img.pixels[fy as usize * w + fx as usize]; + [p.r(), p.g(), p.b()] + } else { + [0x16, 0x17, 0x1c] + }; + for v in c { + out.push_str(&format!("{v:02x}")); + } + } + } + out +} + +pub fn decode_thumb(text: &str) -> Option { + let t = text.trim(); + if t.len() != THUMB_W * THUMB_H * 6 { + return None; + } + let b: Vec = (0..t.len()).step_by(2).map(|i| u8::from_str_radix(&t[i..i + 2], 16)).collect::>().ok()?; + let px = b.chunks_exact(3).map(|c| Color32::from_rgb(c[0], c[1], c[2])).collect(); + Some(egui::ColorImage::new([THUMB_W, THUMB_H], px)) +} + +/// Store the viewer's frame as the open project's thumbnail after it is opened or saved (once per +/// saved revision, when the viewer shows that revision). +pub fn capture_thumbnail(app: &mut EffectcraftApp) { + let s = &app.session; + let (Some(path), Some(_)) = (s.path.clone(), s.config.as_ref()) else { return }; + if s.is_dirty() { + return; + } + let key = (path.clone(), s.saved_revision); + if app.home_thumb_saved.as_ref() == Some(&key) { + return; + } + if app.viewer_shown.as_ref().map(|(_, k)| k.revision) != Some(s.revision) { + return; + } + let Some(img) = app.viewer_pixels() else { return }; + let text = encode_thumb(&img); + if let Some(c) = &app.session.config + && let Err(e) = c.write(&thumb_name(&path), &text) + { + log::warn!("project thumbnail: {e}"); + } + app.home_thumbs.remove(&path); + app.home_thumb_saved = Some(key); +} + +fn thumb_texture(app: &mut EffectcraftApp, ctx: &egui::Context, path: &str) -> Option { + if let Some(t) = app.home_thumbs.get(path) { + return t.clone(); + } + let tex = app + .session + .config + .as_ref() + .and_then(|c| c.read(&thumb_name(path))) + .and_then(|t| decode_thumb(&t)) + .map(|img| ctx.load_texture(format!("home-thumb-{path}"), img, egui::TextureOptions::LINEAR)); + app.home_thumbs.insert(path.to_string(), tex.clone()); + tex +} + +/// Draw the Home screen. +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, Color32::from_rgb(0x1b, 0x1d, 0x24)); + let wide = rect.width() >= 820.0; + let pad = 32.0; + let left_w = if wide { 250.0 } else { rect.width() - pad * 2.0 }; + let mut actions: Vec<(&str, serde_json::Value)> = vec![]; + + // ---- left: brand + start actions. + let x0 = rect.min.x + pad; + let mut y = rect.min.y + pad; + crate::header::paint_logo(&p, Rect::from_min_size(pos2(x0, y), vec2(40.0, 40.0))); + p.text(pos2(x0 + 52.0, y + 12.0), Align2::LEFT_CENTER, "EffectCraft", Tokens::semibold(22.0), Color32::WHITE); + p.text(pos2(x0 + 52.0, y + 32.0), Align2::LEFT_CENTER, format!("Version {}", env!("CARGO_PKG_VERSION")), Tokens::ui(11.0), t.text_faint); + y += 64.0; + for (label, id, primary) in [ + ("New Project", "file.newProject", true), + ("Open Project…", "file.open", false), + ("New Composition", "app.newComp", false), + ("Open Demo Project", "file.openDemoProject", false), + ("Import Footage…", "file.import", false), + ] { + let r = Rect::from_min_size(pos2(x0, y), vec2(left_w.min(250.0), 32.0)); + if widgets::text_button(ui, r, label, primary, &t, egui::Id::new(("home", id))).clicked() { + actions.push((id, json!({}))); + } + app.auto.add(&format!("home.{id}"), r, label); + y += 40.0; + } + + // ---- community (After Effects' Learn / What's New area). + let (cx, mut cy, cw) = if wide { (x0, y + 18.0, left_w) } else { (x0, y + 10.0, left_w) }; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "Community", Tokens::semibold(13.0), t.text); + cy += 16.0; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "Tutorials, help and what's new", Tokens::ui(11.0), t.text_faint); + cy += 16.0; + let links = [ + (Icon::Chat, "Join the ArtCraft Discord", "help.discord"), + (Icon::Globe, "getartcraft.com", "help.website"), + (Icon::Globe, "EffectCraft home page", "help.appPage"), + (Icon::Code, "EffectCraft on GitHub", "help.github"), + ]; + for (icon, label, cmd) in links { + let r = Rect::from_min_size(pos2(cx, cy), vec2(cw.min(250.0), 30.0)); + let resp = ui.interact(r, egui::Id::new(("home-link", cmd)), Sense::click()); + let discord = cmd == "help.discord"; + let bg = if discord { + Color32::from_rgb(0x58, 0x65, 0xf2) + } else if resp.hovered() { + t.hover + } else { + Color32::from_rgb(0x2a, 0x2c, 0x34) + }; + p.rect_filled(r, 15.0, bg); + icons::paint(&p, Rect::from_center_size(pos2(r.min.x + 20.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); + p.text(pos2(r.min.x + 36.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::medium(12.0), Color32::WHITE); + app.auto.add(&format!("home.{cmd}"), r, label); + if resp.clicked() { + let _ = app.session.execute(cmd, json!({})); + } + cy += 36.0; + } + cy += 8.0; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "More ArtCraft apps", Tokens::ui(11.0), t.text_faint); + cy += 12.0; + let sib = effectcraft_engine::links::SIBLINGS; + let per_row = 2usize; + let sw = (cw.min(250.0) - 6.0) / per_row as f32; + for (i, (name, slug)) in sib.iter().enumerate() { + let r = Rect::from_min_size(pos2(cx + (i % per_row) as f32 * (sw + 6.0), cy + (i / per_row) as f32 * 28.0), vec2(sw, 24.0)); + let resp = ui.interact(r, egui::Id::new(("sib", *slug)), Sense::click()); + p.rect_filled(r, 12.0, if resp.hovered() { t.hover } else { Color32::from_rgb(0x24, 0x26, 0x2e) }); + p.text(r.center(), Align2::CENTER_CENTER, *name, Tokens::ui(11.5), t.text); + app.auto.add(&format!("home.sibling.{slug}"), r, name); + if resp.clicked() { + let _ = app.session.execute("help.sibling", json!({"app": slug})); + } + } + let community_bottom = cy + sib.len().div_ceil(per_row) as f32 * 28.0; + + // ---- recent projects. + let (rx, mut ry, rw) = if wide { + let rx = x0 + left_w + 40.0; + (rx, rect.min.y + pad + 6.0, rect.max.x - pad - rx) + } else { + (x0, community_bottom + 24.0, left_w) + }; + p.text(pos2(rx, ry + 6.0), Align2::LEFT_CENTER, "Recent", Tokens::semibold(16.0), Color32::WHITE); + let entries = recent_entries(&app.session.prefs); + if !entries.is_empty() { + let cr = Rect::from_min_size(pos2(rx + rw - 100.0, ry - 4.0), vec2(100.0, 20.0)); + let resp = ui.interact(cr, egui::Id::new("home-clear-recent"), Sense::click()); + p.text(cr.right_center(), Align2::RIGHT_CENTER, "Clear list", Tokens::ui(11.0), if resp.hovered() { t.text } else { t.text_faint }); + app.auto.add("home.clearRecent", cr, "Clear Recent Projects"); + if resp.clicked() { + actions.push(("file.clearRecent", json!({}))); + } + } + ry += 26.0; + // Column headings. + let date_w = 130.0; + p.text(pos2(rx + 112.0, ry), Align2::LEFT_CENTER, "Name", Tokens::ui(11.0), t.text_faint); + p.text(pos2(rx + rw - date_w, ry), Align2::LEFT_CENTER, "Modified", Tokens::ui(11.0), t.text_faint); + ry += 10.0; + p.line_segment([pos2(rx, ry), pos2(rx + rw, ry)], Stroke::new(1.0, t.separator)); + ry += 6.0; + if entries.is_empty() { + p.text(pos2(rx, ry + 20.0), Align2::LEFT_CENTER, "No recent projects. Projects you open or save appear here.", Tokens::ui(12.0), t.text_faint); + } + let row_h = 66.0; + for e in &entries { + let r = Rect::from_min_size(pos2(rx, ry), vec2(rw, row_h - 4.0)); + if r.min.y > rect.max.y { + break; + } + let resp = ui.interact(r, egui::Id::new(("home-recent", e.index)), Sense::click()).on_hover_text(&e.path); + if resp.hovered() { + p.rect_filled(r, 6.0, t.hover); + } + let tr = Rect::from_min_size(pos2(r.min.x + 6.0, r.min.y + 4.0), vec2(THUMB_W as f32, THUMB_H as f32)); + match thumb_texture(app, &ctx, &e.path) { + Some(tex) => { + p.image(tex.id(), tr, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } + None => { + p.rect_filled(tr, 3.0, Color32::from_rgb(0x24, 0x26, 0x2e)); + icons::paint(&p, Rect::from_center_size(tr.center(), vec2(20.0, 20.0)), Icon::Comp, t.text_faint); + } + } + p.rect_stroke(tr, 3.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); + let tx = tr.max.x + 12.0; + let text_clip = p.with_clip_rect(Rect::from_min_max(pos2(tx, r.min.y), pos2(r.max.x - date_w - 8.0, r.max.y))); + text_clip.text(pos2(tx, r.min.y + 20.0), Align2::LEFT_CENTER, &e.name, Tokens::semibold(13.0), if e.exists { Color32::WHITE } else { t.text_dim }); + let sub = if e.exists { e.folder.clone() } else { format!("{} (missing)", e.folder) }; + text_clip.text(pos2(tx, r.min.y + 40.0), Align2::LEFT_CENTER, sub, Tokens::ui(11.0), t.text_faint); + if let Some(d) = &e.modified { + p.text(pos2(r.max.x - date_w, r.min.y + 20.0), Align2::LEFT_CENTER, d, Tokens::ui(11.0), t.text_dim); + } + app.auto.add(&format!("home.recent.{}", e.index), r, &e.path); + if resp.clicked() { + actions.push(("file.openRecent", json!({"index": e.index}))); + } + ry += row_h; + } + + for (id, params) in actions { + let before = (app.session.path.clone(), app.session.revision); + match crate::menus::invoke(app, &ctx, id, params) { + Err(e) => app.ui.status = e, + // Leave Home unless only the list changed or a file dialog was cancelled. + Ok(_) => { + let cancelled = matches!(id, "file.open" | "file.import") && (app.session.path.clone(), app.session.revision) == before; + if id != "file.clearRecent" && !cancelled { + app.ui.start_screen = false; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recent_list_comes_from_settings() { + let mut prefs = effectcraft_engine::prefs::Prefs::default(); + prefs.push_recent("/projects/a/Alpha.ecproj"); + prefs.push_recent("/projects/b/Beta.ecproj"); + let e = recent_entries(&prefs); + assert_eq!(e.len(), 2); + assert_eq!((e[0].name.as_str(), e[0].folder.as_str(), e[0].index), ("Beta", "/projects/b", 0)); + assert_eq!(e[1].name, "Alpha"); + assert!(!e[0].exists && e[0].modified.is_none()); + // A real file has a date. + let dir = std::env::temp_dir().join(format!("ec-home-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("Real.ecproj"); + std::fs::write(&f, "{}").unwrap(); + prefs.push_recent(&f.to_string_lossy()); + let e = recent_entries(&prefs); + assert!(e[0].exists && e[0].modified.as_ref().is_some_and(|d| d.len() == 16), "{:?}", e[0]); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn dates_and_thumbnails() { + assert_eq!(format_utc(0), "1970-01-01 00:00"); + assert_eq!(format_utc(1_790_000_000), "2026-09-21 14:13"); + assert_eq!(format_utc(951_782_400), "2000-02-29 00:00"); + let img = egui::ColorImage::new([200, 100], vec![Color32::from_rgb(10, 200, 30); 200 * 100]); + let text = encode_thumb(&img); + let back = decode_thumb(&text).unwrap(); + assert_eq!(back.size, [THUMB_W, THUMB_H]); + // Letterboxed: the middle is the image, the top row is the background (2:1 into 16:9). + assert_eq!(back.pixels[THUMB_H / 2 * THUMB_W + THUMB_W / 2], Color32::from_rgb(10, 200, 30)); + assert_ne!(back.pixels[THUMB_W / 2], Color32::from_rgb(10, 200, 30)); + assert!(decode_thumb("abc").is_none()); + assert_ne!(thumb_name("/a.ecproj"), thumb_name("/b.ecproj")); + } + + #[test] + fn home_screen_registers_its_controls() { + let mut s = effectcraft_engine::Session::default(); + s.prefs.push_recent("/projects/Alpha.ecproj"); + let mut app = EffectcraftApp::new(s); + let ctx = egui::Context::default(); + crate::theme::install(&ctx, &app.tokens); + ctx.run_ui(Default::default(), |_| {}).textures_delta.clear(); + ctx.run_ui(Default::default(), |ui| { + app.auto.begin_frame(); + show(&mut app, ui, Rect::from_min_size(pos2(0.0, 0.0), vec2(1200.0, 800.0))); + }) + .textures_delta + .clear(); + for id in [ + "home.file.newProject", + "home.file.open", + "home.app.newComp", + "home.file.openDemoProject", + "home.help.discord", + "home.help.github", + "home.recent.0", + ] { + assert!(app.auto.find(id).is_some(), "{id}"); + } + } +} diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 4d8e1e247..0db1ed7f6 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -4,7 +4,7 @@ use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; use serde_json::json; use crate::dock::PanelKind; -use crate::icons::{self, Icon}; +use crate::icons::Icon; use crate::theme::Tokens; use crate::{EffectcraftApp, widgets}; @@ -223,73 +223,3 @@ pub fn markers(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } } - -/// Home screen: new/open, recent, and community links. -pub fn start_screen(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { - let t = app.tokens; - let ctx = ui.ctx().clone(); - let p = ui.painter().with_clip_rect(rect); - p.rect_filled(rect, 0.0, Color32::from_rgb(0x1b, 0x1d, 0x24)); - let c = rect.center(); - let logo = Rect::from_center_size(c + vec2(0.0, -150.0), vec2(64.0, 64.0)); - crate::header::paint_logo(&p, logo); - p.text(c + vec2(0.0, -96.0), Align2::CENTER_CENTER, "EffectCraft", Tokens::semibold(28.0), Color32::WHITE); - p.text(c + vec2(0.0, -66.0), Align2::CENTER_CENTER, "Motion graphics and visual effects, in pure Rust.", Tokens::ui(13.0), t.text_dim); - let mut y = c.y - 30.0; - for (label, id, primary) in [ - ("New Composition", "app.newComp", true), - ("Open Demo Project", "file.openDemoProject", false), - ("Open Project…", "file.open", false), - ("Import Footage…", "file.import", false), - ] { - let r = Rect::from_center_size(pos2(c.x, y + 16.0), vec2(240.0, 32.0)); - if widgets::text_button(ui, r, label, primary, &t, egui::Id::new(("home", id))).clicked() { - let _ = crate::menus::invoke(app, &ctx, id, json!({})); - app.ui.start_screen = false; - } - app.auto.add(&format!("home.{id}"), r, label); - y += 40.0; - } - // Community. - y += 14.0; - let links = [(Icon::Chat, "Join the Discord", "help.discord"), (Icon::Globe, "getartcraft.com", "help.website"), (Icon::Code, "GitHub", "help.github")]; - let w = 170.0; - let mut x = c.x - w * 1.5 - 8.0; - for (icon, label, cmd) in links { - let r = Rect::from_min_size(pos2(x, y), vec2(w, 30.0)); - let resp = ui.interact(r, egui::Id::new(("home-link", cmd)), Sense::click()); - let discord = cmd == "help.discord"; - let bg = if discord { - Color32::from_rgb(0x58, 0x65, 0xf2) - } else if resp.hovered() { - t.hover - } else { - Color32::from_rgb(0x2a, 0x2c, 0x34) - }; - p.rect_filled(r, 15.0, bg); - icons::paint(&p, Rect::from_center_size(pos2(r.min.x + 20.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); - p.text(pos2(r.min.x + 34.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::medium(12.0), Color32::WHITE); - app.auto.add(&format!("home.{cmd}"), r, label); - if resp.clicked() { - let _ = app.session.execute(cmd, json!({})); - } - x += w + 8.0; - } - // Siblings. - y += 52.0; - p.text(pos2(c.x, y), Align2::CENTER_CENTER, "More ArtCraft apps", Tokens::ui(11.5), t.text_faint); - y += 18.0; - let sib = effectcraft_engine::links::SIBLINGS; - let sw = 104.0; - let mut x = c.x - sw * sib.len() as f32 / 2.0; - for (name, slug) in sib { - let r = Rect::from_min_size(pos2(x + 2.0, y), vec2(sw - 4.0, 24.0)); - let resp = ui.interact(r, egui::Id::new(("sib", *slug)), Sense::click()); - p.rect_filled(r, 12.0, if resp.hovered() { t.hover } else { Color32::from_rgb(0x24, 0x26, 0x2e) }); - p.text(r.center(), Align2::CENTER_CENTER, *name, Tokens::ui(11.5), t.text); - if resp.clicked() { - let _ = app.session.execute("help.sibling", json!({"app": slug})); - } - x += sw; - } -} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..dae89f166 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -10,6 +10,7 @@ pub mod forms; pub mod fx_widgets; pub mod graph; pub mod graph_tools; +pub mod home; pub mod info; pub mod key_dialogs; pub mod layer_panel; @@ -63,7 +64,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec match p { PanelKind::Composition => { if app.ui.start_screen { - misc::start_screen(app, ui, rect); + home::show(app, ui, rect); } else { viewer::show(app, ui, rect) } From ed72e67155e17a2b1349b66462b26bccf5ef06bc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:27:22 -0400 Subject: [PATCH 09/14] M6.7: refine edge + freeze + UI Refine Edge tool (Alt+W cycles Roto Brush / Refine Edge), Layer panel roto strokes with matte overlays (Alpha Boundary in pink, Alpha, Alpha Overlay, View Search Region), segmentation span bar, Freeze/Propagate in the Layer panel and Effect Controls, background propagation polling, docs. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 2 +- crates/engine/src/commands/roto_cmds.rs | 4 +- crates/ui-egui/src/icons.rs | 10 + crates/ui-egui/src/lib.rs | 1 + crates/ui-egui/src/menus.rs | 3 +- crates/ui-egui/src/panels/effect_controls.rs | 31 ++ crates/ui-egui/src/panels/layer_panel.rs | 39 ++- crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/roto_tool.rs | 331 +++++++++++++++++++ crates/ui-egui/src/state.rs | 21 +- docs/agents.md | 22 ++ docs/architecture.md | 13 + docs/parity.md | 4 +- 13 files changed, 467 insertions(+), 15 deletions(-) create mode 100644 crates/ui-egui/src/panels/roto_tool.rs diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979cf..55b05822f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -32,6 +32,6 @@ Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 Octobe - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index 5b9db20fa..95326b423 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -378,14 +378,12 @@ fn status(s: &mut Session, p: &Value) -> Result { let (layer, g) = group(s, cid, lid, uid)?; let (layer, g) = (layer.clone(), g.clone()); let d = data_of(&g); - let computed = s.roto_computed(cid, lid, uid); out["layer"] = json!(lid.0); out["effect"] = json!(uid); out["base"] = json!(d.base); out["span"] = json!(d.span); out["strokes"] = json!(d.strokes.len()); out["strokeFrames"] = json!(d.stroke_frames()); - out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); out["frozen"] = json!(frozen_of(&g)); out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); out["limits"] = json!(rt::frame_limits(&comp, &layer)); @@ -420,5 +418,7 @@ fn status(s: &mut Session, p: &Value) -> Result { } } } + let computed = s.roto_computed(cid, lid, uid); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); Ok(out) } diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index d28cb9500..d2de8fc38 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -33,6 +33,7 @@ pub enum Icon { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, // switches & timeline Eye, @@ -282,6 +283,15 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.ellipse(6.0, 10.0, 3.6, 3.6); pen.dot(6.0, 10.0, 1.0); } + RefineEdge => { + // A brush over a soft (dashed) edge. + pen.line(&[(13.5, 2.5), (9.0, 7.0)]); + pen.ellipse(7.0, 9.0, 2.6, 2.6); + pen.line(&[(2.0, 14.0), (3.5, 12.5)]); + pen.line(&[(5.0, 14.0), (6.5, 12.5)]); + pen.line(&[(8.0, 14.0), (9.5, 12.5)]); + pen.line(&[(11.0, 14.0), (12.5, 12.5)]); + } Puppet => { pen.circle(8.0, 3.6, 1.6); pen.line(&[(8.0, 5.2), (8.0, 10.0)]); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..71cb9f454 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -725,6 +725,7 @@ impl EffectcraftApp { // Warp Stabilizer: finish analyses and start queued (re-)analyses in the background. if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() { self.session.poll_warp(true); + self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } self.apply_prefs(&ctx); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2d..b14ade472 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -72,7 +72,7 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("tool.pen", "Pen Tool", [], Some("G")), uic!("tool.type", "Type Tools", [], Some("Cmd+T")), uic!("tool.brush", "Brush Tools", [], Some("Cmd+B")), - uic!("tool.rotoBrush", "Roto Brush Tool", [], Some("Alt+W")), + uic!("tool.rotoBrush", "Roto Brush & Refine Edge Tools", [], Some("Alt+W")), uic!("tool.puppet", "Puppet Tools", [], Some("Cmd+P")), uic!("app.newComp", "New Composition...", [], None), uic!("app.compSettings", "Composition Settings...", [], None), @@ -139,6 +139,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V "pen" => Some(9), "type" => Some(10), "brush" => Some(11), + "rotoBrush" => Some(14), "puppet" => Some(15), _ => None, }; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f7..03afd94cd 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -537,6 +537,35 @@ fn warp_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Roto Brush & Refine Edge: Freeze / Unfreeze and Propagate buttons and the segmentation status. +fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::roto as fx; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = fx::is_frozen(¶ms); + let d = fx::data(¶ms); + let running = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid) && app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, label, frozen, &t, egui::Id::new(("roto-freeze-ec", g.uid))).clicked() && !running { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Propagate", "roto.propagate") }; + if widgets::text_button(ui, b2, l2, false, &t, egui::Id::new(("roto-prop-ec", g.uid))).clicked() && !frozen { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.propagate", g.uid), b2, l2); + let status = match (app.session.roto_progress(), d.base) { + (Some(pr), _) if running => pr.banner(), + (_, None) => "No strokes: paint with the Roto Brush tool (Alt+W) in the Layer panel".to_string(), + (_, Some(b)) => format!("Base frame {b}, span {}–{}{}", d.span[0], d.span[1], if frozen { " (frozen)" } else { "" }), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -545,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::roto::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +950,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index e53e268e7..c20a0fba5 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -3,9 +3,13 @@ //! //! The image is the layer's source and masks followed by its effects up to the View menu's //! choice (by default the last Paint effect, as After Effects shows "Paint"). Dragging with a -//! paint tool records the stroke's points in layer space and, on release, runs `paint.stroke` +//! paint tools records the stroke's points in layer space and, on release, runs `paint.stroke` //! at the current time (the drag's duration feeds Write On). Alt/Option-click with the Clone //! Stamp sets the clone source. +//! +//! With a Roto Brush & Refine Edge effect the panel shows the effect's input with the matte +//! overlay, the segmentation span bar and the view / Freeze buttons; the Roto Brush and Refine +//! Edge tools paint their strokes here (see [`super::roto_tool`]). use effectcraft_engine::effects::paint; use effectcraft_engine::project::{Layer, LayerId}; @@ -46,9 +50,15 @@ pub fn current_layer(app: &EffectcraftApp) -> Option { comp.layer(id).cloned() } -/// Number of effects shown by default: through the last Paint effect (none when there isn't one). -fn default_view(layer: &Layer) -> usize { - layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()).unwrap_or(0) +/// Number of effects shown by default: the Roto Brush's input while a Roto Brush tool is active +/// (or when there is no Paint effect), else through the last Paint effect (none when there +/// isn't one). +fn default_view(layer: &Layer, tool: Tool) -> usize { + let paint = layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()); + match super::roto_tool::roto_index(layer) { + Some(r) if tool.roto_kind(false).is_some() || paint.is_none() => r, + _ => paint.unwrap_or(0), + } } pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { @@ -64,7 +74,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp = app.session.project.comp(cid).cloned().unwrap_or_else(|| effectcraft_engine::project::Comp::new(1, 1, Default::default(), Default::default())); let time = app.session.time(); let fx_count = layer.effects().map(|f| f.groups().count()).unwrap_or(0); - let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer)); + let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer, app.ui.tool)); + let roto = super::roto_tool::roto_index(&layer); // Title strip. let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); @@ -90,9 +101,17 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let tc = super::timecode(&app.session, &comp, time); p.text(pos2(bar.max.x - 10.0, bar.center().y), Align2::RIGHT_CENTER, tc, Tokens::mono(12.0), t.timecode); + // Roto Brush: view / Freeze buttons and the segmentation span bar. + let mut canvas_bottom = bar.min.y; + if roto.is_some() { + super::roto_tool::bar_buttons(app, ui, &p, bar, vr.max.x + 12.0, &layer); + let sb = Rect::from_min_max(pos2(rect.min.x, bar.min.y - 18.0), pos2(rect.max.x, bar.min.y)); + super::roto_tool::span_bar(app, ui, &p, sb, &comp, &layer); + canvas_bottom = sb.min.y; + } // Canvas. - let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, bar.min.y)); + let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, canvas_bottom)); p.rect_filled(area, 0.0, t.pasteboard); let (w, h) = effectcraft_engine::render::source_size(&app.session.project, &layer); let (lw, lh) = if w == 0 { (comp.width as f64, comp.height as f64) } else { (w as f64, h as f64) }; @@ -139,10 +158,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_max(to_screen([tx.rect[0], tx.rect[1]]), to_screen([tx.rect[2], tx.rect[3]])); painter.image(tx.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } + if roto.is_some_and(|r| r == view) { + super::roto_tool::draw_overlay(app, &ctx, &painter, &comp, &layer, scale, &to_screen); + } painter.rect_stroke(frame, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); app.auto.add("layerPanel.canvas", frame, &layer.name); app.auto.add("layerPanel.area", area, "Layer panel"); + // Interaction: Roto Brush / Refine Edge. + if app.ui.tool.roto_kind(false).is_some() { + super::roto_tool::interact(app, ui, &painter, area, &layer, zoom, &to_screen, &to_layer); + return; + } // Interaction: paint tools. let resp = ui.interact(area, egui::Id::new("layer-panel-interact"), Sense::click_and_drag()); let kind = app.ui.tool.paint_kind(); diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..8a8ce0cf9 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -21,6 +21,7 @@ pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; +pub mod roto_tool; pub mod settings; pub mod shortcut_editor; pub mod text_panels; diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs new file mode 100644 index 000000000..25354ee34 --- /dev/null +++ b/crates/ui-egui/src/panels/roto_tool.rs @@ -0,0 +1,331 @@ +//! Roto Brush and Refine Edge tools in the Layer panel: green foreground strokes, Alt/Option red +//! background strokes, Refine Edge strokes along soft edges; the matte overlay (Alpha Boundary in +//! pink, Alpha, Alpha Overlay), the segmentation span bar and the Freeze button. + +use effectcraft_engine::effects::roto as fx; +use effectcraft_engine::project::{Comp, Layer}; +use effectcraft_engine::render::{EvalCtx, RenderOpts, Renderer}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde_json::json; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// Position of the layer's first Roto Brush & Refine Edge effect. +pub fn roto_index(layer: &Layer) -> Option { + layer.effects()?.groups().position(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == fx::ID)) +} + +fn roto_group(layer: &Layer) -> Option<&effectcraft_engine::project::PropGroup> { + layer.effects()?.groups().nth(roto_index(layer)?) +} + +/// A stroke being drawn. +#[derive(Clone, Debug)] +struct Drag { + layer: u64, + kind: &'static str, + points: Vec<[f64; 2]>, +} + +#[derive(Clone)] +struct Overlay { + key: (u64, u64, i64, u32, String), + tex: egui::TextureHandle, + rect: [f64; 4], +} + +fn c32(c: [f64; 3], a: u8) -> Color32 { + Color32::from_rgba_unmultiplied((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8, a) +} + +/// Draw the matte overlay of the Roto Brush on `layer` (the panel shows the effect's input). +pub fn draw_overlay( + app: &mut EffectcraftApp, + ctx: &egui::Context, + painter: &egui::Painter, + comp: &Comp, + layer: &Layer, + scale: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, +) { + let Some(idx) = roto_index(layer) else { return }; + let o = app.session.state.roto.clone(); + if o.view == "none" { + return; + } + let Some(cid) = app.session.active_comp_id() else { return }; + let time = app.session.time(); + let key = ( + app.session.revision, + layer.id.0, + time.0, + (scale * 1000.0) as u32, + format!("{}{:?}{:?}{}", o.view, o.overlay_color, o.boundary_color, o.overlay_opacity), + ); + let id = egui::Id::new("roto-overlay"); + let cached: Option = ctx.data(|d| d.get_temp(id)); + let ov = match cached { + Some(c) if c.key == key => Some(c), + _ => { + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref() }; + let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); + r.expr = app.session.expr.as_deref(); + r.cache = Some(&app.session.layer_cache); + let (Some(inp), Some(out)) = (r.layer_input(&ectx, layer, idx), r.layer_input(&ectx, layer, idx + 1)) else { return }; + let (w, h) = (inp.img.width as usize, inp.img.height as usize); + let a_at = |x: i64, y: i64| -> f32 { + // Output pixel at the same layer point. + let lx = (x as f64 + 0.5 - inp.offset[0]) / inp.scale; + let ly = (y as f64 + 0.5 - inp.offset[1]) / inp.scale; + let (ox, oy) = out.to_px([lx, ly]); + out.img.get(ox.floor() as i64, oy.floor() as i64)[3] + }; + let alpha: Vec = (0..w * h).map(|i| a_at((i % w) as i64, (i / w) as i64)).collect(); + let inside = |x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && alpha[y as usize * w + x as usize] >= 0.5; + let mut px = vec![Color32::TRANSPARENT; w * h]; + let bc = c32(o.boundary_color, 255); + let oc = c32(o.overlay_color, (o.overlay_opacity / 100.0 * 255.0).clamp(0.0, 255.0) as u8); + // View Search Region: the band around the boundary (yellow). + let search = roto_group(layer).filter(|g| g.prop("rotoBrushMatte/viewSearchRegion").is_some_and(|p| p.value.as_bool())).map(|g| { + let r = g.prop("rotoBrushMatte/searchRadius").map(|p| p.value.as_f64()).unwrap_or(15.0) * scale; + let bin: Vec = alpha.iter().map(|a| (*a >= 0.5) as u8).collect(); + let sd = effectcraft_engine::track::roto::matting::signed_distance(&bin, w, h); + (sd, r as f32) + }); + for y in 0..h as i64 { + for x in 0..w as i64 { + let i = y as usize * w + x as usize; + let a = alpha[i]; + px[i] = match o.view.as_str() { + "alpha" => { + let v = (a.clamp(0.0, 1.0) * 255.0) as u8; + Color32::from_rgb(v, v, v) + } + "alphaOverlay" if a < 0.5 => oc, + "alphaOverlay" => Color32::TRANSPARENT, + _ => { + let me = a >= 0.5; + let edge = me && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside(x + dx, y + dy)); + if edge { bc } else { Color32::TRANSPARENT } + } + }; + if let Some((sd, r)) = &search { + let d = sd[i].abs(); + if (d - r).abs() < 0.75 { + px[i] = Color32::from_rgb(255, 220, 0); + } + } + } + } + let img = egui::ColorImage { size: [w, h], pixels: px, source_size: egui::Vec2::new(w as f32, h as f32) }; + let x0 = -inp.offset[0] / inp.scale; + let y0 = -inp.offset[1] / inp.scale; + let rect = [x0, y0, x0 + w as f64 / inp.scale, y0 + h as f64 / inp.scale]; + let tex = ctx.load_texture("roto-overlay", img, egui::TextureOptions::NEAREST); + let c = Overlay { key, tex, rect }; + ctx.data_mut(|d| d.insert_temp(id, c.clone())); + Some(c) + } + }; + if let Some(ov) = ov { + let r = Rect::from_min_max(to_screen([ov.rect[0], ov.rect[1]]), to_screen([ov.rect[2], ov.rect[3]])); + painter.image(ov.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } +} + +/// Pointer handling for the Roto Brush / Refine Edge tools. Returns true when it handled input. +pub fn interact( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + area: Rect, + layer: &Layer, + zoom: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, + to_layer: &dyn Fn(Pos2) -> [f64; 2], +) -> bool { + let ctx = ui.ctx().clone(); + let mods = ui.input(|i| i.modifiers); + let Some(kind) = app.ui.tool.roto_kind(mods.alt) else { return false }; + let resp = ui.interact(area, egui::Id::new("layer-panel-roto"), Sense::click_and_drag()); + let o = app.session.state.roto.clone(); + let refine = kind.starts_with("refine"); + let dia = if refine { o.refine_diameter } else { o.diameter }; + let col = match kind { + "fg" => Color32::from_rgb(40, 220, 60), + "bg" => Color32::from_rgb(235, 50, 40), + "refine" => Color32::from_rgb(110, 200, 255), + _ => Color32::from_rgb(255, 160, 60), + }; + if let Some(hp) = resp.hover_pos() { + let lp = to_layer(hp); + app.pointer_comp = Some([lp[0] as f32, lp[1] as f32]); + ctx.set_cursor_icon(egui::CursorIcon::Crosshair); + let r = (dia * 0.5 * zoom) as f32; + painter.circle_stroke(hp, r, Stroke::new(1.5, col)); + let sign = if kind == "bg" || kind == "refineErase" { "−" } else { "+" }; + painter.text(hp, Align2::CENTER_CENTER, sign, Tokens::semibold(12.0), col); + } + let did = egui::Id::new("layer-panel-roto-drag"); + if resp.drag_started() + && let Some(pos) = ui.input(|i| i.pointer.press_origin()).or(resp.interact_pointer_pos()) + { + ctx.data_mut(|d| d.insert_temp(did, Drag { layer: layer.id.0, kind, points: vec![to_layer(pos)] })); + } + if let Some(mut d) = ctx.data(|d| d.get_temp::(did)) { + if let Some(pos) = resp.interact_pointer_pos() { + let lp = to_layer(pos); + let last = d.points.last().copied().unwrap_or(lp); + if (lp[0] - last[0]).hypot(lp[1] - last[1]) * zoom >= 1.5 { + d.points.push(lp); + ctx.data_mut(|dd| dd.insert_temp(did, d.clone())); + } + } + let width = ((dia * zoom) as f32).max(1.0); + let pts: Vec = d.points.iter().map(|q| to_screen(*q)).collect(); + let c = col.gamma_multiply(0.6); + if pts.len() == 1 { + painter.circle_filled(pts[0], width / 2.0, c); + } else { + painter.add(egui::Shape::line(pts, Stroke::new(width, c))); + } + if resp.drag_stopped() { + ctx.data_mut(|dd| dd.remove::(did)); + run_stroke(app, d.layer, d.kind, &d.points, dia / 2.0); + } + } else if resp.clicked() + && let Some(pos) = resp.interact_pointer_pos() + { + run_stroke(app, layer.id.0, kind, &[to_layer(pos)], dia / 2.0); + } + true +} + +fn run_stroke(app: &mut EffectcraftApp, layer: u64, kind: &str, points: &[[f64; 2]], radius: f64) { + let r = app.session.execute("roto.stroke", json!({"layer": layer, "kind": kind, "points": points, "radius": radius})); + if let Err(e) = r { + app.ui.status = e.to_string(); + } +} + +/// The segmentation span bar (above the panel's bottom bar): the layer's frames, the span, the +/// base frame (yellow), computed frames (green) and stroke frames. Click to go to a frame; drag +/// the span's ends to change it. +pub fn span_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, r: Rect, comp: &Comp, layer: &Layer) { + let t = app.tokens; + let Some(g) = roto_group(layer) else { return }; + let Some(cid) = app.session.active_comp_id() else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let d = fx::data(¶ms); + p.rect_filled(r, 0.0, t.panel_bg); + let lim = effectcraft_engine::roto::frame_limits(comp, layer); + let n = (lim[1] - lim[0] + 1).max(1) as f32; + let track = Rect::from_min_max(pos2(r.min.x + 10.0, r.min.y + 4.0), pos2(r.max.x - 10.0, r.max.y - 4.0)); + let x_of = |f: f64| track.min.x + ((f - lim[0] as f64) / n as f64) as f32 * track.width(); + let f_of = |x: f32| (lim[0] as f64 + ((x - track.min.x) / track.width()).clamp(0.0, 0.9999) as f64 * n as f64).floor() as i64; + p.rect_filled(track, 2.0, t.pasteboard); + if let Some(base) = d.base { + let sp = Rect::from_min_max(pos2(x_of(d.span[0] as f64), track.min.y), pos2(x_of(d.span[1] as f64 + 1.0), track.max.y)); + let frozen = fx::is_frozen(¶ms); + p.rect_filled(sp, 2.0, if frozen { Color32::from_rgb(60, 110, 190) } else { Color32::from_rgb(90, 90, 96) }); + for (f, done) in app.session.roto_computed(cid, layer.id, uid) { + if done { + let cr = Rect::from_min_max(pos2(x_of(f as f64), track.max.y - 4.0), pos2(x_of(f as f64 + 1.0), track.max.y)); + p.rect_filled(cr, 0.0, Color32::from_rgb(70, 200, 90)); + } + } + for f in d.stroke_frames() { + let x = x_of(f as f64 + 0.5); + p.line_segment([pos2(x, track.min.y), pos2(x, track.min.y + 5.0)], Stroke::new(2.0, Color32::WHITE)); + } + let bx = x_of(base as f64 + 0.5); + p.rect_filled(Rect::from_center_size(pos2(bx, track.center().y), vec2(6.0, track.height())), 1.0, Color32::from_rgb(240, 200, 40)); + } + // Current time. + let fps = comp.frame_rate.as_f64(); + let cur = fx::frame_of(layer.layer_time(app.session.time()).seconds(), fps); + let cx = x_of(cur as f64 + 0.5); + p.line_segment([pos2(cx, r.min.y), pos2(cx, r.max.y)], Stroke::new(1.0, t.timecode)); + p.rect_stroke(track, 2.0, Stroke::new(1.0, Color32::from_black_alpha(140)), StrokeKind::Outside); + app.auto.add("layerPanel.rotoSpan", track, "Segmentation span"); + let resp = ui.interact(track, egui::Id::new("roto-span-bar"), Sense::click_and_drag()); + let hid = egui::Id::new("roto-span-handle"); + if resp.drag_started() + && let (Some(pos), Some(_)) = (resp.interact_pointer_pos(), d.base) + { + let (a, b) = (x_of(d.span[0] as f64), x_of(d.span[1] as f64 + 1.0)); + let h = if (pos.x - a).abs() < 6.0 { + 1 + } else if (pos.x - b).abs() < 6.0 { + 2 + } else { + 0 + }; + ui.ctx().data_mut(|m| m.insert_temp(hid, h)); + } + let handle: i32 = ui.ctx().data(|m| m.get_temp(hid)).unwrap_or(0); + if let Some(pos) = resp.interact_pointer_pos() { + let f = f_of(pos.x); + if handle == 0 && (resp.clicked() || resp.dragged()) { + let t = layer.comp_time(comp.frame_rate.tick_of(f)); + let _ = app.session.execute("time.set", json!({"time": t.seconds()})); + } + if handle != 0 && resp.dragged() { + p.line_segment([pos2(pos.x, track.min.y), pos2(pos.x, track.max.y)], Stroke::new(2.0, Color32::from_rgb(240, 200, 40))); + } + if handle != 0 && resp.drag_stopped() { + let k = if handle == 1 { "start" } else { "end" }; + if let Err(e) = app.session.execute("roto.span", json!({"layer": layer.id.0, "effect": uid, k: f})) { + app.ui.status = e.to_string(); + } + } + } + if resp.drag_stopped() { + ui.ctx().data_mut(|m| m.remove::(hid)); + } +} + +/// View-mode toggles and Freeze / Propagate buttons in the Layer panel's bottom bar, from `x`. +pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, bar: Rect, x: f32, layer: &Layer) { + let t = app.tokens; + let mut x = x; + let view = app.session.state.roto.view.clone(); + for (id, label) in [("alpha", "Alpha"), ("alphaBoundary", "Boundary"), ("alphaOverlay", "Overlay")] { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(66.0, 20.0)); + let on = view == id; + if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("roto-view", id))).clicked() { + let v = if on { "none" } else { id }; + let _ = app.session.execute("roto.options", json!({"view": v})); + } + app.auto.add(&format!("layerPanel.roto.view.{id}"), r, label); + x += 70.0; + } + let Some(g) = roto_group(layer) else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let frozen = fx::is_frozen(¶ms); + let running = app.session.is_roto_running(); + x += 8.0; + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(74.0, 20.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, r, label, frozen, &t, egui::Id::new(("roto-freeze", uid))).clicked() { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + if let Err(e) = app.session.execute(id, json!({"layer": layer.id.0, "effect": uid})) { + app.ui.status = e.to_string(); + } + } + app.auto.add("layerPanel.roto.freeze", r, label); + x += 80.0; + if running { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0)); + if widgets::text_button(ui, r, "Stop", false, &t, egui::Id::new("roto-stop")).clicked() { + let _ = app.session.execute("roto.cancel", json!({})); + } + app.auto.add("layerPanel.roto.stop", r, "Stop"); + if let Some(pr) = app.session.roto_progress() { + p.text(pos2(r.max.x + 8.0, bar.center().y), Align2::LEFT_CENTER, pr.banner(), Tokens::ui(11.5), t.text_dim); + } + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d13501032..b89ddadb3 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -34,6 +34,7 @@ pub enum Tool { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, PuppetStarch, PuppetBend, @@ -58,10 +59,10 @@ impl Tool { &[Tool::Brush], &[Tool::Clone], &[Tool::Eraser], - &[Tool::RotoBrush], + &[Tool::RotoBrush, Tool::RefineEdge], &[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap], ]; - pub const ALL: [Tool; 29] = [ + pub const ALL: [Tool; 30] = [ Tool::Selection, Tool::Hand, Tool::Zoom, @@ -86,6 +87,7 @@ impl Tool { Tool::Clone, Tool::Eraser, Tool::RotoBrush, + Tool::RefineEdge, Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, @@ -118,6 +120,7 @@ impl Tool { Tool::Clone => "Clone Stamp Tool", Tool::Eraser => "Eraser Tool", Tool::RotoBrush => "Roto Brush Tool", + Tool::RefineEdge => "Refine Edge Tool", Tool::Puppet => "Puppet Position Pin Tool", Tool::PuppetStarch => "Puppet Starch Pin Tool", Tool::PuppetBend => "Puppet Bend Pin Tool", @@ -139,7 +142,7 @@ impl Tool { Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"), Tool::Type | Tool::TypeVertical => Some("Cmd+T"), Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"), - Tool::RotoBrush => Some("Alt+W"), + Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"), Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"), } } @@ -169,6 +172,7 @@ impl Tool { Tool::Clone => Icon::Clone, Tool::Eraser => Icon::Eraser, Tool::RotoBrush => Icon::RotoBrush, + Tool::RefineEdge => Icon::RefineEdge, Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Icon::Puppet, } } @@ -187,6 +191,17 @@ impl Tool { _ => return None, }) } + /// Roto Brush / Refine Edge: the stroke kind they paint (`roto.stroke` kind), without and + /// with Alt/Option. + pub fn roto_kind(self, alt: bool) -> Option<&'static str> { + Some(match (self, alt) { + (Tool::RotoBrush, false) => "fg", + (Tool::RotoBrush, true) => "bg", + (Tool::RefineEdge, false) => "refine", + (Tool::RefineEdge, true) => "refineErase", + _ => return None, + }) + } /// Paint tools, with their stroke kind (`paint.stroke` kind). pub fn paint_kind(self) -> Option<&'static str> { Some(match self { diff --git a/docs/agents.md b/docs/agents.md index 510bc356d..5c1b41876 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,28 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### Roto Brush & Refine Edge + +The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use +`roto.stroke {layer, kind: fg|bg|refine|refineErase, points: [[x, y], …], frame?, radius?}` +(layer pixels, layer frames). The first foreground stroke applies the Roto Brush & Refine Edge +effect and sets the base frame with a span of 20 frames each side. `roto.propagate {layer, +direction?: forward|backward|both, to?, wait?}` segments the span (in the background unless +`wait`); strokes on any other frame correct it and propagation restarts from there. +`roto.span {start?, end?}`, `roto.freeze {wait?}` / `roto.unfreeze`, `roto.clearStrokes {frame?, +kind?}`, `roto.cancel` and `roto.options {diameter?, refineDiameter?, view?: alphaBoundary|alpha| +alphaOverlay|none}` complete the set; every edit is one undo step. `roto.status {layer, frame?, +matte?, compute?, compareTo?}` reports the base frame, span, computed and stroked frames, frozen +state, job progress and, for a frame, the matte's area, centroid, RLE matte and IoU against a +reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/searchRadius`, +`effects/#1/refineEdgeMatte/decontaminateEdgeColors`, …). + +```sh +effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[360,230]],"radius":10}' \ + roto.stroke '{"layer":"#1","kind":"bg","points":[[40,40],[600,40]],"radius":12}' \ + roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb5..9f1216080 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,19 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`): +strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes +parameter with the base frame and segmentation span. Each frame is segmented by graph cut +(Boykov–Jolly hard constraints, GrabCut colour mixtures, our own Boykov–Kolmogorov max-flow, +coarse to fine) and propagated to the next frame by warping the matte with block optical flow and +re-cutting in a Search Radius band; frames with correction strokes are re-cut with the warped +matte as a soft prior. Refine Edge bands get guided-filter + closed-form matting and +foreground-colour decontamination. Segmentations are derived data: each frame's chain key (Input +Key, settings, strokes from the base frame out) indexes a process-wide cache that renders fill on +demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` keeps the Input +Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes +in the effect. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec24..6e926dc71 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems (Roto Brush and Refine Edge landed in M6.7): the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -97,7 +97,7 @@ wasm threads. | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | -| Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | +| Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | From 406706e3d55b47f6addf342d1ddc08db12ff2b09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:28:15 -0400 Subject: [PATCH 10/14] OUT-2: WebM export (VP9 intra encoder + alpha, Opus), WAV/AIFF audio-only outputs - effectcraft-vp9enc (L0): VP9 profile 0 key-frame encoder from the VP9 bitstream spec (8x8 intra blocks, DC/V/H/TM, 4x4 DCT/ADST, lossless WHT at quality 100, tile columns); round-trips exactly through FilmCraft's VP9 decoder and ffmpeg. - export: WebM muxer (EBML/Matroska spec) with VP9, VP9 alpha in BlockAdditional (AlphaMode 1), Opus (CodecDelay = pre-skip); WAV and AIFF outputs. - OutputFormat WebM/Wav/Aiff in the Render Queue, renderQueue.add formats, CLI --format. - Tests: FilmCraft decode-back (frames, size, pixels, audio RMS), ffprobe/ffmpeg oracles. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 5 + apps/effectcraft-cli/src/main.rs | 2 +- crates/engine/src/commands/render_queue.rs | 8 +- crates/engine/src/rq_tests.rs | 2 +- crates/export/Cargo.toml | 4 +- crates/export/src/lib.rs | 6 + crates/export/src/webm.rs | 368 +++++++++ crates/export/tests/roundtrip.rs | 92 +++ crates/export/tests/vp9_roundtrip.rs | 149 ++++ crates/host/tests/vector_import.rs | 23 + crates/media/src/convert.rs | 1 - crates/project/src/render_queue.rs | 30 +- crates/ui-egui/src/panels/render_queue.rs | 2 +- crates/vp9enc/Cargo.toml | 12 + crates/vp9enc/README.md | 15 + crates/vp9enc/src/bool.rs | 193 +++++ crates/vp9enc/src/lib.rs | 857 +++++++++++++++++++++ crates/vp9enc/src/tables.rs | 130 ++++ docs/architecture.md | 15 +- docs/parity.md | 10 +- xtask/src/main.rs | 1 + 22 files changed, 1911 insertions(+), 20 deletions(-) create mode 100644 crates/export/src/webm.rs create mode 100644 crates/export/tests/vp9_roundtrip.rs create mode 100644 crates/vp9enc/Cargo.toml create mode 100644 crates/vp9enc/README.md create mode 100644 crates/vp9enc/src/bool.rs create mode 100644 crates/vp9enc/src/lib.rs create mode 100644 crates/vp9enc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 866ae15f1..9b28b8ace 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,11 +1226,13 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1485,6 +1487,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 08a7d19d6..dc2ce073c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } @@ -71,6 +72,7 @@ filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf2 filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -140,6 +142,9 @@ opt-level = 3 [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66c..3a04e15cd 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,7 +9,7 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c2..be6cebfb7 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -131,7 +131,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -256,7 +256,7 @@ fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; it.output = OutputModule::for_format(f); } apply_settings(&mut it.settings, p, "renderQueue.add")?; @@ -434,7 +434,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { let n = it.extra_outputs.len() + 2; let mut om = it.output.clone(); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om = OutputModule::for_format(f); } if str_p(p, "output").is_none() && str_p(p, "path").is_none() { @@ -509,7 +509,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), diff --git a/crates/engine/src/rq_tests.rs b/crates/engine/src/rq_tests.rs index c4aa08b8a..f172b5e97 100644 --- a/crates/engine/src/rq_tests.rs +++ b/crates/engine/src/rq_tests.rs @@ -159,6 +159,6 @@ fn without_exporter() { s.execute("renderQueue.add", json!({})).unwrap(); assert!(!s.is_enabled("renderQueue.render")); let f = s.execute("renderQueue.formats", json!({})).unwrap(); - assert_eq!(f["formats"].as_array().unwrap().len(), 7); + assert_eq!(f["formats"].as_array().unwrap().len(), 10); assert_eq!(f["formats"][0]["available"], false); } diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 5bdd0fee6..15f9b17d2 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -19,6 +19,7 @@ filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } effectcraft-opusenc = { workspace = true } +effectcraft-vp9enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -30,3 +31,4 @@ log = { workspace = true } effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } +filmcraft-vp9 = { workspace = true } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c83..a2af6a4f4 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,6 +7,8 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | +//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -18,6 +20,7 @@ mod encode; mod out; +mod webm; use web_time::Instant; @@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< let mut report = match job.output.format { OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs new file mode 100644 index 000000000..eb99cd227 --- /dev/null +++ b/crates/export/src/webm.rs @@ -0,0 +1,368 @@ +//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF. +//! +//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML, +//! the Matroska element registry and the WebM container guidelines): EBML header, Segment +//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are +//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose +//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every +//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit +//! extended sample rate) follow their published file layouts. + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +const OPUS_RATE: u32 = 48_000; +const OPUS_BITRATE: u32 = 192_000; + +// Element ids (with their length markers). +const EBML: u32 = 0x1A45_DFA3; +const SEGMENT: u32 = 0x1853_8067; +const INFO: u32 = 0x1549_A966; +const TRACKS: u32 = 0x1654_AE6B; +const TRACK_ENTRY: u32 = 0xAE; +const CLUSTER: u32 = 0x1F43_B675; + +fn id_bytes(id: u32) -> Vec { + let b = id.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(3); + b[skip..].to_vec() +} + +/// EBML variable-size integer for an element data size (shortest form). +fn vint(n: u64) -> Vec { + for len in 1..=8u32 { + let max = (1u64 << (7 * len)) - 2; + if n <= max { + let marked = n | (1u64 << (7 * len)); + return marked.to_be_bytes()[8 - len as usize..].to_vec(); + } + } + let mut v = vec![0x01]; + v.extend_from_slice(&n.to_be_bytes()[1..]); + v +} + +fn el(out: &mut Vec, id: u32, data: &[u8]) { + out.extend(id_bytes(id)); + out.extend(vint(data.len() as u64)); + out.extend_from_slice(data); +} + +fn el_uint(out: &mut Vec, id: u32, v: u64) { + let b = v.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(7); + el(out, id, &b[skip..]); +} + +fn el_float(out: &mut Vec, id: u32, v: f64) { + el(out, id, &v.to_be_bytes()); +} + +fn el_str(out: &mut Vec, id: u32, s: &str) { + el(out, id, s.as_bytes()); +} + +/// A block payload: track number, timestamp relative to the cluster, flags, frame. +fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { + let mut b = vint(track); + b.extend_from_slice(&rel.to_be_bytes()); + b.push(flags); + b.extend_from_slice(data); + b +} + +struct Frame { + ms: i64, + track: u64, + data: Vec, + alpha: Option>, +} + +fn cluster(frames: &mut [Frame]) -> Vec { + frames.sort_by_key(|f| (f.ms, f.track)); + let base = frames.first().map_or(0, |f| f.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for f in frames.iter() { + let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + match &f.alpha { + None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + Some(a) => { + let mut g = vec![]; + el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + let mut more = vec![]; + el_uint(&mut more, 0xEE, 1); + el(&mut more, 0xA5, a); + let mut adds = vec![]; + el(&mut adds, 0xA6, &more); + el(&mut g, 0x75A1, &adds); + el(&mut body, 0xA0, &g); + } + } + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let alpha = job.output.channels == Channels::Rgba; + let quality = job.output.quality.clamp(1, 100); + let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + // EBML header. + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + // Segment with an 8-byte size patched at the end. + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_VP9"); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + if alpha { + el_uint(&mut v, 0x55EE, 1); + } + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + if alpha { + el_uint(&mut video, 0x53C0, 1); + } + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + // Audio: mixed in blocks, encoded in 20 ms packets. + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + // Pad the last packet (pre-skip + padding cover the encoder delay). + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + while i < total { + let end = (i + batch).min(total); + let encoded: Vec<(Vec, Option>)> = (i..end) + .into_par_iter() + .map(|k| { + let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); + let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); + let color = enc.encode_yuv420(&y, &u, &vv); + let a = alpha.then(|| { + let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); + ea.encode_yuv420(&y, &u, &vv) + }); + (color, a) + }) + .collect(); + let mut frames: Vec = + encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut frames, end >= total); + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + st.advance(end - i)?; + i = end; + } + if total == 0 { + let mut frames = vec![]; + audio_until(span_end, &mut frames, true); + if !frames.is_empty() { + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + } + } + // Patch the Segment size. + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} + +// ---------------------------------------------------------------- audio-only outputs + +fn ieee_extended(v: f64) -> [u8; 10] { + // 80-bit IEEE 754 extended precision (AIFF COMM sample rate). + let mut out = [0u8; 10]; + if v <= 0.0 { + return out; + } + let e = v.log2().floor() as i32; + let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64; + out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. +pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { + let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let (span_start, span_end) = job.settings.span(comp); + let start = span_start.to_units_floor(sr as i64); + let end = span_end.to_units_floor(sr as i64); + let n = (end - start).max(0) as usize; + let mut pcm = Vec::with_capacity(n * 4); + // Mix in one-second blocks, reporting progress in frames. + let block = sr as usize; + let mut done = 0usize; + let total_frames = st.total.max(1); + let mut reported = 0u64; + while done < n { + let k = block.min(n - done); + let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); + for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { + let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; + pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); + } + done += k; + let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); + if now > reported && st.total > 0 { + st.advance(now - reported)?; + reported = now; + } + } + if reported < st.total { + st.advance(st.total - reported)?; + } + let mut out = vec![]; + let frames = (pcm.len() / 4) as u32; + if aiff { + let mut comm = vec![]; + comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&frames.to_be_bytes()); + comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&ieee_extended(sr as f64)); + let ssnd_len = 8 + pcm.len(); + out.extend_from_slice(b"FORM"); + out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes()); + out.extend_from_slice(b"AIFF"); + out.extend_from_slice(b"COMM"); + out.extend_from_slice(&(comm.len() as u32).to_be_bytes()); + out.extend_from_slice(&comm); + out.extend_from_slice(b"SSND"); + out.extend_from_slice(&(ssnd_len as u32).to_be_bytes()); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&pcm); + if ssnd_len % 2 == 1 { + out.push(0); + } + } else { + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&sr.to_le_bytes()); + out.extend_from_slice(&(sr * 4).to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); + out.extend_from_slice(&pcm); + } + let mut file = crate::out::create(job.sink, job.path)?; + file.write_all(&out).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vints_and_ids() { + assert_eq!(vint(0), vec![0x80]); + assert_eq!(vint(126), vec![0xFE]); + assert_eq!(vint(127), vec![0x40, 0x7F]); + assert_eq!(id_bytes(0xA3), vec![0xA3]); + assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]); + assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]); + assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]); + } +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 186aa1a97..1ade594f8 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -363,3 +363,95 @@ fn movies_carry_audio() { assert!(!effectcraft_media::probe(&path).expect("probe").has_audio); } } + +#[test] +fn webm_vp9_opus_roundtrip() { + let d = out_dir("webm"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("comp.webm"); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio && r.bytes > 200, "{r:?}"); + // Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders). + let f = decode_movie(&path, FRAMES, 0.08); + assert!(f.has_audio); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + let l = rms(s.iter().step_by(2).copied()); + assert!((l - 0.3536).abs() < 0.06, "left RMS {l}"); + ffmpeg_check(&path, 0.08); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("vp9|64|48|yuv420p"), "{s}"); + assert!(s.contains("opus"), "{s}"); + } +} + +#[test] +fn webm_with_alpha() { + let d = out_dir("webm-alpha"); + let (p, cid) = project(None); + // Transparent outside the red solid. + let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("alpha.webm"); + run(&p, cid, &NoFootage, &om, &path); + let f = effectcraft_media::probe(&path).expect("probe our output"); + assert_eq!((f.width, f.height), (W, H)); + let bytes = std::fs::read(&path).unwrap(); + // AlphaMode element (0x53C0) and BlockAdditions (0x75A1). + assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1"); + assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions"); + // ffmpeg's libvpx decoder reads the alpha channel. + let png = path.with_extension("alpha.png"); + let ok = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"]) + .arg(&path) + .args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"]) + .arg(&png) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + let img = image::open(&png).unwrap().to_rgba8(); + assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent"); + assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque"); + } else { + eprintln!("ffmpeg with libvpx not available: skipping alpha oracle"); + } +} + +#[test] +fn wav_and_aiff_audio_only() { + let d = out_dir("audio-only"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] { + let om = OutputModule::for_format(fmt); + let path = d.join(name); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..4], magic); + // 1.2 s of 48 kHz 16-bit stereo. + assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len()); + let l = if fmt == OutputFormat::Wav { + // Decoded by FilmCraft. + let f = effectcraft_media::probe(&path).expect("probe audio"); + assert!(f.has_audio && !f.has_video, "{name}"); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + rms(s.iter().step_by(2).copied()) + } else { + // AIFF: big-endian samples after the SSND chunk's offset/block size. + let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16; + let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0); + rms(left) + }; + assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}"); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("pcm_s16"), "{s}"); + } + } +} diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs new file mode 100644 index 000000000..8156d8505 --- /dev/null +++ b/crates/export/tests/vp9_roundtrip.rs @@ -0,0 +1,149 @@ +//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and +//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR. + +use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder}; +use filmcraft_vp9::{Decoder, Plane}; + +fn plane_u8(p: &Plane) -> Vec { + match p { + Plane::U8(v) => v.clone(), + Plane::U16(v) => v.iter().map(|x| *x as u8).collect(), + } +} + +/// Synthetic planes: gradients, noise and sharp edges. +fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut s = 99u32 + kind; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 24) as u8 + }; + let y: Vec = (0..w * h) + .map(|i| { + let (x, yy) = (i % w, i / w); + match kind { + 0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2, + 1 => rnd(), + _ => { + if (x / 7 + yy / 5) % 2 == 0 { + 30 + } else { + 220 + } + } + } + }) + .collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect(); + (y, u, v) +} + +fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { + let mut d = Decoder::new(); + let mut pics = d.decode(frame, 0).expect("decode"); + assert_eq!(pics.len(), 1); + pics.remove(0) +} + +fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec { + (0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect() +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::() / a.len() as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +#[test] +fn lossless_is_exact_for_odd_sizes() { + for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { + for kind in 0..3 { + let (y, u, v) = source(w, h, kind); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert_eq!((p.width, p.height), (w as u32, h as u32)); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma"); + assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u"); + assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v"); + } + } +} + +#[test] +fn lossy_quality_and_size() { + let (w, h) = (200, 120); + for kind in [0, 2] { + let (y, u, v) = source(w, h, kind); + let mut last = 0usize; + for q in [30u8, 60, 80, 95] { + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + let got = crop(&plane_u8(&p.y), p.y_stride, w, h); + let db = psnr(&got, &y); + let min = match q { + 30 => 20.0, + 60 => 28.0, + 80 => 32.0, + _ => 38.0, + }; + assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len()); + assert!(f.len() >= last, "higher quality, more bytes"); + last = f.len(); + } + } +} + +#[test] +fn wide_frames_use_tile_columns() { + // 4160 px > 64 superblocks: two tile columns are required. + let (w, h) = (4160, 16); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); +} + +#[test] +fn ffmpeg_decodes_our_stream() { + let (w, h) = (96, 64); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let frame = e.encode_yuv420(&y, &u, &v); + // IVF container (one frame). + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&0u64.to_le_bytes()); + ivf.extend_from_slice(&frame); + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); + assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); +} diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index eb5be076b..590112c9f 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -161,3 +161,26 @@ fn svg_footage_and_shapes_from_vector_layer() { assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); } } + +#[test] +fn svg_footage_continuously_rasterizes() { + let path = tmp("cr.svg"); + std::fs::write(&path, SVG).unwrap(); + let doc = effectcraft_svg::parse(SVG).unwrap(); + let (w, h) = doc.pixel_size(); + let mut s = effectcraft_host::session(); + let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); + assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); +} diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index dce33d8ac..1edcf5fd4 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -392,4 +392,3 @@ mod tests { } } } - diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5a..d1e6683c1 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -136,17 +136,26 @@ pub enum OutputFormat { ExrSequence, /// Animated GIF. Gif, + /// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio. + WebM, + /// Audio only: WAV (16-bit PCM). + Wav, + /// Audio only: AIFF (16-bit PCM). + Aiff, } impl OutputFormat { - pub const ALL: [OutputFormat; 7] = [ + pub const ALL: [OutputFormat; 10] = [ OutputFormat::H264, OutputFormat::ProRes, + OutputFormat::WebM, OutputFormat::PngSequence, OutputFormat::JpegSequence, OutputFormat::TiffSequence, OutputFormat::ExrSequence, OutputFormat::Gif, + OutputFormat::Wav, + OutputFormat::Aiff, ]; pub fn label(self) -> &'static str { @@ -158,6 +167,9 @@ impl OutputFormat { OutputFormat::TiffSequence => "TIFF Sequence", OutputFormat::ExrSequence => "OpenEXR Sequence", OutputFormat::Gif => "Animated GIF", + OutputFormat::WebM => "WebM (VP9 + Opus)", + OutputFormat::Wav => "WAV", + OutputFormat::Aiff => "AIFF", } } pub fn extension(self) -> &'static str { @@ -169,19 +181,26 @@ impl OutputFormat { OutputFormat::TiffSequence => "tif", OutputFormat::ExrSequence => "exr", OutputFormat::Gif => "gif", + OutputFormat::WebM => "webm", + OutputFormat::Wav => "wav", + OutputFormat::Aiff => "aif", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + } + /// Audio-only outputs (no video is rendered). + pub fn is_audio_only(self) -> bool { + matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence) + !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { - self.is_movie() + self.is_movie() || self.is_audio_only() } /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, /// at least 2). @@ -201,6 +220,9 @@ impl OutputFormat { "tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence, "exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence, "gif" | "animatedgif" => OutputFormat::Gif, + "webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM, + "wav" | "wave" => OutputFormat::Wav, + "aif" | "aiff" | "aifc" => OutputFormat::Aiff, _ => return None, }) } diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2d..06403181e 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -108,7 +108,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 000000000..62144b258 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 000000000..c8f781b87 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 000000000..45db92dde --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 000000000..02cbb0e28 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 000000000..dc8027a6e --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/architecture.md b/docs/architecture.md index c3d869c01..096c75bda 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,18 +16,21 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | | L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | @@ -76,6 +79,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* diff --git a/docs/parity.md b/docs/parity.md index d259fec24..5d291ca70 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -75,7 +75,7 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, +the new tracking library, a JavaScript scripting object model, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -84,9 +84,9 @@ wasm threads. | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | @@ -98,7 +98,7 @@ wasm threads. | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -125,7 +125,7 @@ Mocha and Cineware are third-party and not counted. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). 12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7e7616a5e..d822a1b96 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,6 +23,7 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), ("opusenc", 0), ("raster", 1), From 541c740f1859b7e9cf072b868590b07b0e7cea85 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:30:04 -0400 Subject: [PATCH 11/14] CMP-6: Flowchart panel Composition > Composition Flowchart charts the active comp as a node graph: nested comps, footage and solids, optionally the layers and their effects (panel switches Solids / Layers / Effects), left to right or top to bottom. Double-click a comp to open it, Cmd/Ctrl-click to reveal an item in the Project panel (a layer: select it), drag or scroll to pan. Reveal Layer in Project Flowchart shows layers with the layer selected. The graph builder is pure and cycle-safe (shared items drawn once, back edges to comps on the path). Commands flowchart.options {layers, effects, solids, direction, comp} and flowchart.graph (the graph as JSON); automation ids flowchart.node. and flowchart.option.. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/menus.rs | 10 + crates/ui-egui/src/panels/flowchart.rs | 460 +++++++++++++++++++++++++ crates/ui-egui/src/panels/misc.rs | 2 +- crates/ui-egui/src/panels/mod.rs | 3 +- crates/ui-egui/src/state.rs | 4 + 5 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 crates/ui-egui/src/panels/flowchart.rs diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 4c85162e5..da5eb8423 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -72,6 +72,8 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("timeline.column", "Show/Hide Timeline Column", [], None), uic!("timeline.sourceName", "Source Name / Layer Name", [], None), uic!("timeline.search", "Search Timeline", [], None), + uic!("flowchart.options", "Flowchart Options", [], None), + uic!("flowchart.graph", "Flowchart Graph", [], None), uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")), uic!("tool.selection", "Selection Tool", [], Some("V")), uic!("tool.hand", "Hand Tool", [], Some("H")), @@ -331,6 +333,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V tl.source_name = params.get("value").and_then(Value::as_bool).unwrap_or(!tl.source_name); return Ok(json!({"sourceName": tl.source_name})); } + "flowchart.options" | "flowchart.graph" => return crate::panels::flowchart::command(app, id, ¶ms), "timeline.search" => { app.ui.timeline.search = params.get("query").and_then(Value::as_str).unwrap_or_default().to_string(); app.show_panel(PanelKind::Timeline); @@ -712,6 +715,13 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu Value::Null } "comp.flowchart" => { + // Chart the active comp; Layer ▸ Reveal ▸ Reveal Layer in Project Flowchart shows + // layers and selects the layer's node. + app.ui.flowchart.root = app.session.active_comp_id().map(|c| c.0); + if let (Some(c), Some(l)) = (app.session.active_comp_id(), app.session.state.selected_layers.first()) { + app.ui.flowchart.layers = true; + app.ui.flowchart.selected = Some(format!("layer:{}:{}", c.0, l.0)); + } app.show_panel(PanelKind::Flowchart); Value::Null } diff --git a/crates/ui-egui/src/panels/flowchart.rs b/crates/ui-egui/src/panels/flowchart.rs new file mode 100644 index 000000000..2ba345cf2 --- /dev/null +++ b/crates/ui-egui/src/panels/flowchart.rs @@ -0,0 +1,460 @@ +//! The Flowchart panel (Composition ▸ Composition Flowchart): a node graph of a composition and +//! everything it uses — nested comps, footage, solids — optionally with its layers and their +//! effects. Left to right or top to bottom; double-click a comp to open it; Cmd/Ctrl-click a +//! node to reveal it in the Project panel (layers: select the layer); drag to pan. +//! +//! [`build`] is pure and cycle-safe (a comp nested in itself is drawn once, with an edge back), +//! so agents get the same graph through `flowchart.graph`. + +use std::collections::{BTreeMap, BTreeSet}; + +use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, Project}; +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::icons::{self, Icon}; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// Flowchart panel options (the panel's switches). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FlowOptions { + /// Show layers between a comp and its sources. + pub layers: bool, + /// Show each layer's effects (with layers shown). + pub effects: bool, + /// Show solids. + pub solids: bool, + /// Top to bottom instead of left to right. + pub vertical: bool, + /// The comp the chart starts from (None = the active comp). + pub root: Option, + #[serde(skip)] + pub pan: [f32; 2], + /// Selected node id. + #[serde(skip)] + pub selected: Option, +} + +impl Default for FlowOptions { + fn default() -> Self { + FlowOptions { layers: false, effects: false, solids: true, vertical: false, root: None, pan: [0.0, 0.0], selected: None } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FlowNode { + /// `item:`, `layer::` or `fx:::`. + pub id: String, + /// `comp`, `footage`, `solid`, `layer` or `effect`. + pub kind: &'static str, + pub label: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub item: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub comp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub layer: Option, + /// Distance from the root (column, or row when vertical). + pub depth: usize, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FlowGraph { + pub nodes: Vec, + /// (from, to) node ids: from a comp to what it uses. + pub edges: Vec<(String, String)>, +} + +/// Build the flowchart of `root` (a comp). Shared items appear once; cycles stop at the comp +/// already on the path. +pub fn build(project: &Project, root: ItemId, opts: &FlowOptions) -> FlowGraph { + struct B<'a> { + p: &'a Project, + o: &'a FlowOptions, + nodes: Vec, + index: BTreeMap, + edges: Vec<(String, String)>, + stack: BTreeSet, + } + impl B<'_> { + fn node(&mut self, n: FlowNode) -> String { + let id = n.id.clone(); + match self.index.get(&id) { + Some(&i) => self.nodes[i].depth = self.nodes[i].depth.min(n.depth), + None => { + self.index.insert(id.clone(), self.nodes.len()); + self.nodes.push(n); + } + } + id + } + fn edge(&mut self, a: &str, b: &str) { + let e = (a.to_string(), b.to_string()); + if !self.edges.contains(&e) { + self.edges.push(e); + } + } + fn item(&mut self, id: ItemId, depth: usize) -> Option { + let it = self.p.item(id)?; + let kind = match &it.kind { + ItemKind::Comp(_) => "comp", + ItemKind::Footage(_) => "footage", + ItemKind::Solid(_) => "solid", + ItemKind::Folder => return None, + }; + if kind == "solid" && !self.o.solids { + return None; + } + let seen = self.index.contains_key(&format!("item:{}", id.0)); + let nid = self.node(FlowNode { id: format!("item:{}", id.0), kind, label: it.name.clone(), item: Some(id.0), comp: None, layer: None, depth }); + if kind == "comp" && !seen && !self.stack.contains(&id.0) { + self.comp(id, depth); + } + Some(nid) + } + fn comp(&mut self, cid: ItemId, depth: usize) { + let Some(c) = self.p.comp(cid) else { return }; + self.stack.insert(cid.0); + let from = format!("item:{}", cid.0); + for l in &c.layers { + let src_depth = if self.o.layers { depth + 2 } else { depth + 1 }; + let mut parent = from.clone(); + if self.o.layers { + let lid = self.node(FlowNode { + id: format!("layer:{}:{}", cid.0, l.id.0), + kind: "layer", + label: l.name.clone(), + item: None, + comp: Some(cid.0), + layer: Some(l.id.0), + depth: depth + 1, + }); + self.edge(&from, &lid); + if self.o.effects + && let Some(fx) = l.effects() + { + for g in fx.groups() { + let fid = self.node(FlowNode { + id: format!("fx:{}:{}:{}", cid.0, l.id.0, g.uid), + kind: "effect", + label: g.name.clone(), + item: None, + comp: Some(cid.0), + layer: Some(l.id.0), + depth: depth + 2, + }); + self.edge(&lid, &fid); + } + } + parent = lid; + } + let src = match l.source { + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => Some(item), + _ => None, + }; + if let Some(item) = src { + // A comp already on the path: link back to it, don't descend (cycle). + if self.stack.contains(&item.0) { + self.edge(&parent, &format!("item:{}", item.0)); + } else if let Some(sid) = self.item(item, src_depth) { + self.edge(&parent, &sid); + } + } + } + self.stack.remove(&cid.0); + } + } + let mut b = B { p: project, o: opts, nodes: vec![], index: BTreeMap::new(), edges: vec![], stack: BTreeSet::new() }; + b.item(root, 0); + FlowGraph { nodes: b.nodes, edges: b.edges } +} + +const NODE_W: f32 = 168.0; +const NODE_H: f32 = 34.0; + +/// Node positions (top-left, before panning): one column (or row) per depth, in discovery order. +pub fn layout(g: &FlowGraph, vertical: bool) -> Vec<[f32; 2]> { + let mut slot: BTreeMap = BTreeMap::new(); + g.nodes + .iter() + .map(|n| { + let k = slot.entry(n.depth).or_insert(0); + let i = *k as f32; + *k += 1; + let d = n.depth as f32; + if vertical { [i * (NODE_W + 24.0), d * (NODE_H + 46.0)] } else { [d * (NODE_W + 60.0), i * (NODE_H + 14.0)] } + }) + .collect() +} + +/// The comp the panel charts. +pub fn root(app: &EffectcraftApp) -> Option { + app.ui.flowchart.root.map(ItemId).filter(|r| app.session.project.comp(*r).is_some()).or_else(|| app.session.active_comp_id()) +} + +/// `flowchart.options` / `flowchart.graph` (UI commands). +pub fn command(app: &mut EffectcraftApp, id: &str, p: &Value) -> Result { + let o = &mut app.ui.flowchart; + let b = |k: &str| p.get(k).and_then(Value::as_bool); + if let Some(v) = b("layers") { + o.layers = v; + } + if let Some(v) = b("effects") { + o.effects = v; + } + if let Some(v) = b("solids") { + o.solids = v; + } + if let Some(d) = p.get("direction").and_then(Value::as_str) { + o.vertical = match d { + "lr" | "leftToRight" => false, + "tb" | "topToBottom" => true, + _ => return Err(format!("direction: lr|tb, not `{d}`")), + }; + } + if let Some(c) = p.get("comp") { + let cid = match c { + Value::Number(n) => n.as_u64().map(ItemId), + Value::String(s) => app.session.project.find_by_name(s).map(|i| i.id), + _ => None, + }; + let cid = cid.filter(|c| app.session.project.comp(*c).is_some()).ok_or("comp: a composition id or name")?; + app.ui.flowchart.root = Some(cid.0); + } + let o = app.ui.flowchart.clone(); + if id == "flowchart.graph" { + let r = root(app).ok_or("no composition")?; + return Ok(serde_json::to_value(build(&app.session.project, r, &o)).unwrap_or_default()); + } + Ok(json!({"layers": o.layers, "effects": o.effects, "solids": o.solids, "direction": if o.vertical { "tb" } else { "lr" }, "root": o.root})) +} + +fn kind_style(kind: &str) -> (Icon, Color32) { + match kind { + "comp" => (Icon::Comp, Color32::from_rgb(0x3d, 0x5a, 0x8c)), + "footage" => (Icon::Footage, Color32::from_rgb(0x3c, 0x6e, 0x58)), + "solid" => (Icon::Solid, Color32::from_rgb(0x6a, 0x58, 0x3a)), + "effect" => (Icon::Fx, Color32::from_rgb(0x6c, 0x44, 0x6e)), + _ => (Icon::Null, Color32::from_rgb(0x44, 0x48, 0x52)), + } +} + +/// Draw the Flowchart panel. +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.tl_bg); + let Some(rid) = root(app) else { + p.text(rect.center(), Align2::CENTER_CENTER, "(no composition)", Tokens::ui(12.0), t.text_faint); + return; + }; + // ---- toolbar: switches and direction. + let bar = Rect::from_min_size(rect.min, vec2(rect.width(), 30.0)); + p.rect_filled(bar, 0.0, t.panel_bg); + let root_name = app.session.project.item(rid).map(|i| i.name.clone()).unwrap_or_default(); + p.text(pos2(bar.min.x + 10.0, bar.center().y), Align2::LEFT_CENTER, format!("Flowchart: {root_name}"), Tokens::medium(12.0), t.text); + let mut x = bar.max.x - 8.0; + let mut cmds: Vec<(String, Value)> = vec![]; + let o = app.ui.flowchart.clone(); + for (key, label, on) in [ + ("direction", if o.vertical { "Top to Bottom" } else { "Left to Right" }, false), + ("effects", "Effects", o.effects), + ("layers", "Layers", o.layers), + ("solids", "Solids", o.solids), + ] { + let w = 18.0 + label.len() as f32 * 6.6; + let r = Rect::from_min_size(pos2(x - w, bar.min.y + 5.0), vec2(w, 20.0)); + x -= w + 6.0; + if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("flow-opt", key))).clicked() { + let v = if key == "direction" { json!({"direction": if o.vertical { "lr" } else { "tb" }}) } else { json!({key: !on}) }; + cmds.push(("flowchart.options".into(), v)); + } + app.auto.add(&format!("flowchart.option.{key}"), r, label); + } + + // ---- graph. + let body = Rect::from_min_max(pos2(rect.min.x, bar.max.y), rect.max); + let g = build(&app.session.project, rid, &o); + let pos = layout(&g, o.vertical); + let origin = body.min + vec2(24.0 + o.pan[0], 24.0 + o.pan[1]); + let bg = ui.interact(body, egui::Id::new("flow-bg"), Sense::click_and_drag()); + if bg.dragged() { + let d = bg.drag_delta(); + app.ui.flowchart.pan[0] += d.x; + app.ui.flowchart.pan[1] += d.y; + } + if ui.rect_contains_pointer(body) { + let s = ui.input(|i| i.smooth_scroll_delta); + if s != egui::Vec2::ZERO { + app.ui.flowchart.pan[0] += s.x; + app.ui.flowchart.pan[1] += s.y; + } + } + if bg.clicked() { + app.ui.flowchart.selected = None; + } + let gp = p.with_clip_rect(body); + let rect_of = |i: usize| Rect::from_min_size(origin + vec2(pos[i][0], pos[i][1]), vec2(NODE_W, NODE_H)); + let idx: BTreeMap<&str, usize> = g.nodes.iter().enumerate().map(|(i, n)| (n.id.as_str(), i)).collect(); + // Edges first (curves with an arrowhead at the target). + for (a, b) in &g.edges { + let (Some(&ia), Some(&ib)) = (idx.get(a.as_str()), idx.get(b.as_str())) else { continue }; + let (ra, rb) = (rect_of(ia), rect_of(ib)); + let back = g.nodes[ib].depth <= g.nodes[ia].depth; + let (s, e, n) = if o.vertical { (ra.center_bottom(), rb.center_top(), vec2(0.0, 1.0)) } else { (ra.right_center(), rb.left_center(), vec2(1.0, 0.0)) }; + let k = if o.vertical { (e.y - s.y).abs() * 0.5 } else { (e.x - s.x).abs() * 0.5 }.max(30.0); + let col = if back { Color32::from_rgb(0xd0, 0x80, 0x40) } else { t.text_faint }; + let curve = egui::epaint::CubicBezierShape::from_points_stroke([s, s + n * k, e - n * k, e], false, Color32::TRANSPARENT, Stroke::new(1.2, col)); + gp.add(curve); + let (l, r) = if o.vertical { (vec2(-4.0, -7.0), vec2(4.0, -7.0)) } else { (vec2(-7.0, -4.0), vec2(-7.0, 4.0)) }; + gp.add(egui::Shape::convex_polygon(vec![e, e + l, e + r], col, Stroke::NONE)); + } + let cmd_down = ui.input(|i| i.modifiers.command || i.modifiers.ctrl); + for (i, n) in g.nodes.iter().enumerate() { + let r = rect_of(i); + if !r.intersects(body) { + continue; + } + let (icon, fill) = kind_style(n.kind); + let selected = o.selected.as_deref() == Some(n.id.as_str()); + let resp = ui.interact(r.intersect(body), egui::Id::new(("flow-node", &n.id)), Sense::click()); + let fill = if resp.hovered() { fill.gamma_multiply(1.25) } else { fill }; + gp.rect_filled(r, 5.0, fill); + gp.rect_stroke( + r, + 5.0, + Stroke::new(if selected { 2.0 } else { 1.0 }, if selected { t.accent } else { Color32::from_black_alpha(120) }), + StrokeKind::Inside, + ); + icons::paint(&gp, Rect::from_center_size(pos2(r.min.x + 16.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); + gp.with_clip_rect(r.shrink(2.0).intersect(body)).text( + pos2(r.min.x + 30.0, r.center().y), + Align2::LEFT_CENTER, + &n.label, + Tokens::ui(12.0), + Color32::WHITE, + ); + app.auto.add(&format!("flowchart.node.{}", n.id), r, &n.label); + if resp.clicked() { + app.ui.flowchart.selected = Some(n.id.clone()); + if cmd_down { + // Reveal: items in the Project panel, layers selected in their comp. + match (n.item, n.comp, n.layer) { + (Some(item), ..) => { + cmds.push(("project.select".into(), json!({"items": [item]}))); + cmds.push(("window.panel".into(), json!({"panel": "project"}))); + } + (None, Some(c), Some(l)) => { + cmds.push(("comp.open".into(), json!({"comp": c}))); + cmds.push(("layer.select".into(), json!({"layers": [l]}))); + } + _ => {} + } + } + } + if resp.double_clicked() { + match (n.kind, n.item, n.comp) { + ("comp", Some(c), _) => cmds.push(("comp.open".into(), json!({"comp": c}))), + (_, _, Some(c)) => cmds.push(("comp.open".into(), json!({"comp": c}))), + _ => {} + } + } + resp.on_hover_text(format!("{} — double-click to open, {}-click to reveal", n.label, if cfg!(target_os = "macos") { "Cmd" } else { "Ctrl" })); + } + for (id, params) in cmds { + if let Err(e) = crate::menus::invoke(app, &ctx, &id, params) { + app.ui.status = e; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use effectcraft_engine::Session; + + /// Main ⊃ Mid ⊃ Inner (with a solid), Main also uses Inner directly. + fn session() -> (Session, ItemId, ItemId, ItemId) { + let mut s = Session::default(); + let id = |v: Value| ItemId(v["id"].as_u64().or_else(|| v["comp"].as_u64()).or_else(|| v["item"].as_u64()).unwrap()); + let inner = id(s.execute("comp.new", json!({"name": "Inner", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.newSolid", json!({"name": "Red", "color": "#ff0000", "width": 100, "height": 100})).unwrap(); + s.execute("layer.select", json!({"layers": []})).unwrap(); + let mid = id(s.execute("comp.new", json!({"name": "Mid", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + let main = id(s.execute("comp.new", json!({"name": "Main", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.addItem", json!({"item": mid.0})).unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + s.execute("layer.newText", json!({"text": "Title"})).unwrap(); + (s, main, mid, inner) + } + + #[test] + fn nested_comps_make_a_graph() { + let (s, main, mid, inner) = session(); + let g = build(&s.project, main, &FlowOptions::default()); + let label = |id: &str| g.nodes.iter().find(|n| n.id == id).map(|n| (n.label.clone(), n.depth)); + assert_eq!(label(&format!("item:{}", main.0)), Some(("Main".into(), 0))); + assert_eq!(label(&format!("item:{}", mid.0)), Some(("Mid".into(), 1))); + // Inner is reached at depth 1 (directly) and 2 (through Mid): drawn once, at depth 1. + assert_eq!(label(&format!("item:{}", inner.0)), Some(("Inner".into(), 1))); + assert_eq!(g.nodes.iter().filter(|n| n.label == "Inner").count(), 1); + let has = |a: ItemId, b: ItemId| g.edges.contains(&(format!("item:{}", a.0), format!("item:{}", b.0))); + assert!(has(main, mid) && has(mid, inner) && has(main, inner)); + // The solid is shown (Solids on) and hidden with Solids off; text layers have no source. + assert!(g.nodes.iter().any(|n| n.kind == "solid")); + let g2 = build(&s.project, main, &FlowOptions { solids: false, ..Default::default() }); + assert!(!g2.nodes.iter().any(|n| n.kind == "solid")); + // Layers between comps and sources; effects under layers. + let g3 = build(&s.project, main, &FlowOptions { layers: true, ..Default::default() }); + assert!(g3.nodes.iter().any(|n| n.kind == "layer" && n.label == "Title")); + let mid_layer = g3.nodes.iter().find(|n| n.kind == "layer" && n.comp == Some(main.0) && n.label == "Mid").unwrap(); + assert!(g3.edges.contains(&(mid_layer.id.clone(), format!("item:{}", mid.0)))); + // Layout: one column per depth, no overlaps. + let pos = layout(&g3, false); + for i in 0..pos.len() { + for j in i + 1..pos.len() { + assert!(pos[i] != pos[j], "{} and {} overlap", g3.nodes[i].label, g3.nodes[j].label); + } + } + } + + #[test] + fn cycles_are_safe() { + let (mut s, main, mid, _) = session(); + // Make Mid contain Main: Main → Mid → Main. + s.execute("comp.open", json!({"comp": mid.0})).unwrap(); + // Adding a cyclic layer may be refused by the engine; build a cycle in the model directly. + let mut p = (*s.project).clone(); + let mut c = (*p.comp(mid).unwrap()).clone(); + let mut l = c.layers[0].clone(); + l.id = effectcraft_engine::project::LayerId(9999); + l.source = LayerSource::Comp { item: main }; + c.layers.push(l); + if let Some(ItemKind::Comp(arc)) = p.items.get_mut(&mid).map(|i| &mut i.kind) { + *arc = std::sync::Arc::new(c); + } + let g = build(&p, main, &FlowOptions { layers: true, effects: true, ..Default::default() }); + assert_eq!(g.nodes.iter().filter(|n| n.kind == "comp" && n.label == "Main").count(), 1); + // The back edge points at the root. + assert!(g.edges.iter().any(|(_, b)| *b == format!("item:{}", main.0))); + assert!(g.nodes.len() < 50); + } + + #[test] + fn options_command() { + let (s, main, ..) = session(); + let mut app = EffectcraftApp::new(s); + let r = command(&mut app, "flowchart.options", &json!({"layers": true, "direction": "tb"})).unwrap(); + assert_eq!(r["direction"], "tb"); + assert!(app.ui.flowchart.layers && app.ui.flowchart.vertical); + assert!(command(&mut app, "flowchart.options", &json!({"direction": "diagonal"})).is_err()); + let g = command(&mut app, "flowchart.graph", &json!({"comp": "Main"})).unwrap(); + assert_eq!(g["nodes"][0]["id"], format!("item:{}", main.0)); + assert!(g["edges"].as_array().is_some_and(|e| !e.is_empty())); + } +} diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 0db1ed7f6..69d95f664 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -1,4 +1,4 @@ -//! Smaller panels: Preview, Audio, History, Markers, the Home screen and placeholders. +//! Smaller panels: Preview, Audio, History, Markers and placeholders. use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; use serde_json::json; diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index dae89f166..cc103e84b 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -6,6 +6,7 @@ pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; pub mod effects_presets; +pub mod flowchart; pub mod forms; pub mod fx_widgets; pub mod graph; @@ -91,7 +92,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Layer => layer_panel::show(app, ui, rect), PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), - other => misc::placeholder(app, ui, rect, other), + PanelKind::Flowchart => flowchart::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index af59bb6b7..1d1dcb236 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -584,6 +584,9 @@ pub struct UiState { /// Composition Mini-Flowchart popup position (open when set). #[serde(default)] pub mini_flowchart: Option<[f32; 2]>, + /// Flowchart panel options (layers, effects, solids, direction, root comp). + #[serde(default)] + pub flowchart: crate::panels::flowchart::FlowOptions, } impl Default for UiState { @@ -634,6 +637,7 @@ impl Default for UiState { layer_view: None, anim_tools: AnimToolsState::default(), mini_flowchart: None, + flowchart: Default::default(), } } } From 6aca8fe85c9a2dd76fc02419e3270535ae313556 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:34:32 -0400 Subject: [PATCH 12/14] UI-1: every After Effects workspace Adds Review, Learn, Essential Graphics, Color and Undocked Panels to the built-in workspaces (Libraries, an Adobe service, is left out), lists them all in Window > Workspace in After Effects' order, and gives each its own layout: Review (large viewer, Markers and Project), Learn (Default plus the Home screen with the community links), Essential Graphics (Properties panel full height, type panels), Color (wide Effect Controls, Info), Undocked Panels (Composition docked, the other panels floating). Tests check every AE workspace exists, has its own layout, has each panel once and is in the menu. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/menus.rs | 15 +++-- crates/ui-egui/src/dock.rs | 125 ++++++++++++++++++++++++++++++++++++- crates/ui-egui/src/lib.rs | 6 +- 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 38252a884..14c39dbbc 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -17,7 +17,8 @@ //! //! An entry's shortcut is the command's default shortcut unless the line gives one (needed for //! entries with bound parameters). Adobe-service entries (Team Projects, Libraries, Bridge, Media -//! Encoder, Essential Graphics, Behance, Creative Cloud…) are intentionally absent. +//! Encoder, Behance, Creative Cloud…) are intentionally absent; the Essential Graphics workspace +//! uses the Properties panel. use std::sync::OnceLock; @@ -909,16 +910,22 @@ View Enter Full Screen | view.fullScreen Window Workspace + Default | window.workspace {"name":"Default"} | Shift+F10 + Review | window.workspace {"name":"Review"} + Learn | window.workspace {"name":"Learn"} + Small Screen | window.workspace {"name":"Small Screen"} | Shift+F12 + Standard | window.workspace {"name":"Standard"} | Shift+F11 + --- All Panels | window.workspace {"name":"All Panels"} Animation | window.workspace {"name":"Animation"} - Default | window.workspace {"name":"Default"} | Shift+F10 + Color | window.workspace {"name":"Color"} Effects | window.workspace {"name":"Effects"} + Essential Graphics | window.workspace {"name":"Essential Graphics"} Minimal | window.workspace {"name":"Minimal"} Motion Tracking | window.workspace {"name":"Motion Tracking"} Paint | window.workspace {"name":"Paint"} - Small Screen | window.workspace {"name":"Small Screen"} | Shift+F12 - Standard | window.workspace {"name":"Standard"} | Shift+F11 Text | window.workspace {"name":"Text"} + Undocked Panels | window.workspace {"name":"Undocked Panels"} --- Reset to Saved Layout | window.resetWorkspace Save Changes to this Workspace | window.saveWorkspace diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 99bc7c5e4..9d6d55ccf 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -172,7 +172,43 @@ fn vsplit(size: SplitSize, a: DockNode, b: DockNode) -> DockNode { DockNode::Split { vertical: true, size, a: Box::new(a), b: Box::new(b) } } -pub const WORKSPACES: [&str; 10] = ["Default", "Standard", "Small Screen", "Animation", "Effects", "Motion Tracking", "Paint", "Text", "Minimal", "All Panels"]; +/// The built-in workspaces: After Effects 2026's (Window ▸ Workspace), less Libraries (an Adobe +/// service). Learn shows the Home screen (community links instead of Adobe's tutorials); +/// Essential Graphics uses the Properties panel; Undocked Panels floats the panels over the +/// Composition panel. +pub const WORKSPACES: &[&str] = &[ + "Default", + "Standard", + "Small Screen", + "Animation", + "Effects", + "Motion Tracking", + "Paint", + "Text", + "Minimal", + "All Panels", + "Review", + "Learn", + "Essential Graphics", + "Color", + "Undocked Panels", +]; + +/// The floating panels a workspace starts with (Undocked Panels), in points for a window of +/// about 1680×1020 (they are kept inside the window when drawn). +pub fn workspace_floating(name: &str) -> Vec { + use PanelKind::*; + let f = |panels: &[PanelKind], rect: [f32; 4]| Floating { panels: panels.to_vec(), active: 0, rect }; + match name { + "Undocked Panels" => vec![ + f(&[Project, EffectControls], [24.0, 110.0, 320.0, 420.0]), + f(&[Preview, Info, Audio], [1330.0, 110.0, 320.0, 300.0]), + f(&[EffectsPresets, Properties, Character, Paragraph], [1330.0, 430.0, 320.0, 360.0]), + f(&[Timeline, RenderQueue], [24.0, 640.0, 1280.0, 340.0]), + ], + _ => vec![], + } +} /// The default layout of a named workspace. pub fn workspace(name: &str) -> DockNode { @@ -226,6 +262,40 @@ pub fn workspace(name: &str) -> DockNode { tabs(&[Timeline], 0), ), "Minimal" => vsplit(Ratio(0.6), tabs(&[Composition], 0), tabs(&[Timeline], 0)), + // Review: a large viewer with the markers and the project beside it. + "Review" => vsplit( + Ratio(0.66), + hsplit(FixedA(280.0), tabs(&[Project, Markers], 0), hsplit(FixedB(260.0), tabs(&[Composition, Layer], 0), right(&[Preview, Info], &[Audio]))), + tabs(&[Timeline, RenderQueue], 0), + ), + // Essential Graphics: the Properties panel (After Effects' successor to the Essential + // Graphics panel) full height on the right, type panels beside the viewer. + "Essential Graphics" => hsplit( + FixedB(320.0), + vsplit( + Ratio(0.58), + hsplit( + FixedA(280.0), + tabs(&[Project, EffectControls], 0), + hsplit(FixedB(260.0), tabs(&[Composition, Layer], 0), right(&[Character, Paragraph], &[Align, Preview])), + ), + tabs(&[Timeline], 0), + ), + stack(&[(Properties, true, None), (EffectsPresets, false, None)]), + ), + // Color: wide Effect Controls for grading effects, Info for colour readouts. + "Color" => vsplit( + Ratio(0.6), + hsplit( + FixedA(360.0), + tabs(&[EffectControls, Project], 0), + hsplit(FixedB(300.0), tabs(&[Composition, Layer], 0), right(&[Info, Preview], &[EffectsPresets])), + ), + tabs(&[Timeline], 0), + ), + // Undocked Panels: the Composition panel docked, the others floating + // ([`workspace_floating`]). + "Undocked Panels" => tabs(&[Composition, Layer, Flowchart], 0), "Small Screen" => vsplit( Ratio(0.55), hsplit( @@ -814,8 +884,59 @@ mod tests { fn workspaces_contain_core_panels() { for w in WORKSPACES { let d = workspace(w); - assert!(d.contains(PanelKind::Timeline), "{w}"); + let floating: Vec = workspace_floating(w).into_iter().flat_map(|f| f.panels).collect(); + assert!(d.contains(PanelKind::Timeline) || floating.contains(&PanelKind::Timeline), "{w}"); assert!(d.contains(PanelKind::Composition) || d.contains(PanelKind::Layer), "{w}"); + // A panel is in one place only. + let mut all = vec![]; + d.panels(&mut all); + all.extend(floating); + let n = all.len(); + all.sort_by_key(|p| p.id()); + all.dedup(); + assert_eq!(all.len(), n, "{w} has a panel twice"); + } + } + + #[test] + fn after_effects_workspaces_exist() { + // Window ▸ Workspace in After Effects 2026, less Libraries (an Adobe service). + for w in [ + "Default", + "Review", + "Learn", + "Small Screen", + "Standard", + "All Panels", + "Animation", + "Essential Graphics", + "Color", + "Effects", + "Minimal", + "Motion Tracking", + "Paint", + "Text", + "Undocked Panels", + ] { + assert!(WORKSPACES.contains(&w), "{w}"); + } + assert!(!WORKSPACES.contains(&"Libraries")); + // Layouts differ from Default (except Learn, which is Default plus the Home screen). + let def = workspace("Default"); + for w in WORKSPACES.iter().filter(|w| !matches!(**w, "Default" | "Learn")) { + assert_ne!(workspace(w), def, "{w} falls back to Default"); + } + assert!(workspace("Essential Graphics").contains(PanelKind::Properties)); + assert!(workspace("Color").contains(PanelKind::EffectControls)); + assert!(!workspace_floating("Undocked Panels").is_empty()); + // Every built-in workspace is in Window ▸ Workspace. + let menu: Vec = effectcraft_engine::menus::entries() + .into_iter() + .filter(|(_, e)| e.command == "window.workspace") + .filter_map(|(_, e)| e.params.get("name").and_then(|v| v.as_str()).map(str::to_string)) + .collect(); + for w in WORKSPACES { + assert!(menu.iter().any(|m| m == w), "{w} missing from Window > Workspace"); } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 59005cdc6..e1848063c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -279,8 +279,12 @@ impl EffectcraftApp { pub fn set_workspace(&mut self, name: &str) { self.ui.workspace = name.to_string(); self.ui.dock = self.ui.saved_workspaces.get(name).cloned().unwrap_or_else(|| dock::workspace(name)); - self.ui.floating = self.ui.saved_floating.get(name).cloned().unwrap_or_default(); + self.ui.floating = self.ui.saved_floating.get(name).cloned().unwrap_or_else(|| dock::workspace_floating(name)); self.ui.maximized = None; + // Learn: the Home screen (community links) in the Composition panel. + if name == "Learn" { + self.ui.start_screen = true; + } } /// Built-in workspaces followed by the saved ones (Window ▸ Workspace ▸ Save as New Workspace). From 5e441e859df4235427e865adb51044ad58654fa5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:36:30 -0400 Subject: [PATCH 13/14] UI-8, UI-3, UI-7, CMP-6, UI-1: parity and control-protocol docs Co-Authored-By: Claude Opus 5.5 --- docs/control-protocol.md | 1 + docs/parity.md | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 449a18989..7763ee03d 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -27,6 +27,7 @@ printf '{"id":1,"method":"engine.execute","params":{"command":"comp.new","params |---|---|---| | `engine.execute` | `{command, params?}` | The command's result. Every engine command (`effectcraft-cli commands`) runs exactly as it does headless, so it never opens a dialog: `comp.new {}` creates a default comp. UI-only ids (`tool.*`, `view.*`, `window.*`, `playback.*`, `timeline.*`, `app.*`) fall through to the menu dispatcher. Unknown top-level params are rejected, and the error lists the accepted keys. | | `ui.menu.invoke` | `{id, params?}` | Like a menu click: commands with empty params may open their dialog (New Composition, Solid Settings, file pickers). | +| UI commands (via either) | | `timeline.column {column, visible?}` (av, keys, label, num, comment, switches, modes, parent, in, out, duration, stretch), `timeline.sourceName {value?}`, `timeline.search {query}`, `timeline.revealAdd.`, `flowchart.options {layers?, effects?, solids?, direction?: lr\|tb, comp?}`, `flowchart.graph {comp?}` → `{nodes, edges}`, `window.workspace {name}` (all After Effects workspaces). | | `engine.commands` | `{filter?, enabledOnly?}` | `[{id, label, menu, shortcut, params, enabled, why}]` | | `ui.menu.list` | `{}` | The menu tree with enablement. | | `render.frame` | `{comp?, time?, max_side?, path?, base64?}` | Renders a comp frame through the session, independent of the viewer zoom or resolution. `comp` is an id or name (default: the active comp), `time` is in comp seconds (default: the CTI), `max_side` caps the longest side (0 = full size). Writes a PNG to `path` (default: a temp file) and returns `{comp, time, width, height, path}`. With `base64: true` it returns `{…, png: ""}` and writes no file. | diff --git a/docs/parity.md b/docs/parity.md index aecb3f136..0be871b46 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -76,8 +76,9 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, -the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and -wasm threads. +the missing effect categories (3D Channel, Immersive Video, OCIO), and wasm threads. (Native +macOS menus, the Composition Flowchart, Timeline column/search depth, the Home screen and every +After Effects workspace landed in the UI-polish wave.) ## By area @@ -89,13 +90,13 @@ wasm threads. | Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | -| Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | +| Compositions | 72% | 6.0 | Essential Graphics panel (the marker dialog and Composition Flowchart landed) | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | | 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | -| Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | +| Interface | 70% | 6.0 | Timeline horizontal scrolling of many columns, a richer Learn area (native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | @@ -122,7 +123,7 @@ Mocha and Cineware are third-party and not counted. 7. Point tracking and stabilization (in progress). 8. Frame blending, collapse transformations, slip edit. 9. A real 8/16/32-bit pipeline with linear blending and colour management. -10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. +10. ~~Drag-to-dock and floating panels, saved workspaces, native macOS menus.~~ 11. Auto-save, crash recovery, recent projects (in progress). 12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). 13. Lottie (in progress), WebM, SVG and PSD import. From cba8d7e737db9438c9b0e46fd882875df5e7946a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:02:39 -0400 Subject: [PATCH 14/14] M12.6: 3D Camera Tracker effect + UI Effect > Perspective > 3D Camera Tracker (Shot Type, Horizontal Angle of View, Show/Render Track Points, Track Point Size, Target Size, Advanced: Solve Method, Method Used, Average Error, Detailed Analysis, Auto-delete Points Across Time, Hide Warning Banner); tracks and solve stored in the effect with keys (frame edits re-track, setting / deleted-point edits only re-solve). Background job (Analyzing in background (step 1 of 2) / Solving camera banner, cancel, one undo step). Commands: track.camera (replaces the stub), camera.analyze/cancel/solveStatus/points/selectPoints/ createFromSolve/create/setGroundPlane/deletePoints; the one-node 3D Tracker Camera keyed per frame and text / solids / nulls / shadow catcher + light on the target plane, Create Multiple. Viewer: depth-sized coloured targets, hover target plane, click / shift / marquee selection, AE right-click menu, Delete; Effect Controls Analyze / Cancel / Create Camera; Tracker panel Track Camera. Solver: adaptive noise thresholds, denser keyframes, quick tripod check, faster Cholesky and parallel Schur complement. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + ROADMAP.md | 4 +- crates/effects/src/camera_tracker.rs | 291 ++++++++ crates/effects/src/lib.rs | 4 + crates/engine/src/camera_track.rs | 640 ++++++++++++++++++ crates/engine/src/commands/camera_cmds.rs | 574 ++++++++++++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 22 + crates/engine/src/tests_camera_track.rs | 307 +++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/track/Cargo.toml | 1 + crates/track/src/camtrack/bundle.rs | 90 ++- crates/track/src/camtrack/features.rs | 2 +- crates/track/src/camtrack/linalg.rs | 30 +- crates/track/src/camtrack/mod.rs | 8 +- crates/track/src/camtrack/solver.rs | 83 ++- crates/track/src/camtrack/tests.rs | 78 +++ crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 438 ++++++++++++ crates/ui-egui/src/panels/effect_controls.rs | 2 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 7 +- crates/ui-egui/src/panels/viewer.rs | 3 + crates/ui-egui/tests/ui_camera_tracker.rs | 175 +++++ docs/agents.md | 28 + docs/architecture.md | 16 + docs/parity.md | 10 +- 28 files changed, 2771 insertions(+), 54 deletions(-) create mode 100644 crates/effects/src/camera_tracker.rs create mode 100644 crates/engine/src/camera_track.rs create mode 100644 crates/engine/src/commands/camera_cmds.rs create mode 100644 crates/engine/src/tests_camera_track.rs create mode 100644 crates/ui-egui/src/panels/camera_tracker_ui.rs create mode 100644 crates/ui-egui/tests/ui_camera_tracker.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..ce792ecbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979cf..7780c1641 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 258 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Roto Brush, Advanced 3D, a scripting object model (the 3D Camera Tracker landed). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 000000000..dd97b585e --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a1..9c2aefb50 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +pub mod camera_tracker; mod channel; mod channel2; mod color2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 000000000..1b984aac2 --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 000000000..5d454aed5 --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f6..04034281e 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod edit; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd76..74de3aecf 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -59,7 +59,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488ab..fbd2b391e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,6 +172,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -574,6 +594,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 000000000..5bd5539c1 --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858caa..bde98add3 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -331,7 +331,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 9bd46e26c..4800e06eb 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 84e8a5ad6..72cb2f174 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -173,6 +173,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> } let mut cur = cost(cams, pts, obs, o); let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; let mut lambda = 1e-4; let mut iters = 0; if l.n == 0 && o.fix_points { @@ -231,7 +234,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> for (ia, ja) in &jc { let wv = [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; - match ps.w.iter_mut().find(|e| e.0 == *ia) { + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), } @@ -294,31 +300,67 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> // ---- Try steps with increasing damping. let mut accepted = false; for _try in 0..12 { + tries += 1; let mut s = base_s.clone(); let mut rhs = base_bc.clone(); for i in 0..n { let d = s[i * n + i]; s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); } - // Schur complement over the points. - let mut vinv: Vec> = vec![None; npts]; + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; if !o.fix_points { - for (j, ps) in systems.iter().enumerate() { - let mut v = ps.v; - let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; - for k in 0..3 { - v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; - } - let Some(vi) = inv3(&v) else { continue }; - vinv[j] = Some(vi); - let vb = mv(&vi, ps.bp); - let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); - for (a, (ia, wa)) in ps.w.iter().enumerate() { - rhs[*ia] -= dot(*wa, vb); - for (ib, wb) in &ps.w { - s[ia * n + ib] -= dot(wv[a], *wb); + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } } } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); } } let mut dc = rhs; @@ -363,9 +405,18 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> let gain = cur - c2; cur = c2; lambda = (lambda / 3.0).max(1e-9); - accepted = gain > 1e-10 * cur.max(1e-12); + accepted = gain > 1e-6 * cur.max(1e-12); if !accepted { // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } return BaReport { initial_cost: initial, cost: cur, iterations: iters }; } break; @@ -376,6 +427,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> break; } } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } BaReport { initial_cost: initial, cost: cur, iterations: iters } } diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 311ec711d..30b019e54 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -89,7 +89,7 @@ impl TrackAnalyzer { let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); !occ.get(cy * gw + cx).copied().unwrap_or(true) }; - let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; for p in good_features(&pyr, &opts, Some(&free)) { let l = pyr.to_layer(p); let id = self.tracks.len() as u32; diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs index a7067f262..b719cc8e5 100644 --- a/crates/track/src/camtrack/linalg.rs +++ b/crates/track/src/camtrack/linalg.rs @@ -298,6 +298,23 @@ pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { (vals, vecs) } +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + /// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by /// Cholesky factorisation. Returns false when `A` is not positive definite. pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { @@ -313,16 +330,9 @@ pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { a[j * n + j] = d; let (head, tail) = a.split_at_mut((j + 1) * n); let rowj = &head[j * n..j * n + j]; - // Rows below j are independent. - use rayon::prelude::*; - let work = |row: &mut [f64]| { - let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); - row[j] = s / d; - }; - if n - j > 192 { - tail.par_chunks_mut(n).for_each(work); - } else { - tail.chunks_mut(n).for_each(work); + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; } } // Forward: L y = b. diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 056d7ad03..76c0637f6 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -13,7 +13,10 @@ //! the other keyframes are added incrementally (**resection** by robust pose refinement, //! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse //! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points -//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. //! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it //! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal //! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment @@ -22,7 +25,8 @@ //! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame //! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle //! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the -//! tripod solution when it explains the tracks as well as the general one. +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). //! //! The camera model has square pixels, the principal point at the layer centre and no lens //! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index da3a92908..85edd2bd8 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -37,8 +37,9 @@ struct Data { tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, /// Per frame: (track index, position). by_frame: Vec>, - /// Tracking noise scale (pixels). - noise: f64, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, } impl Data { @@ -58,23 +59,31 @@ impl Data { } tracks.push((tr.id, tr.start as usize, pts)); } - Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } } fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { let (_, s, p) = &self.tracks[ti as usize]; k.checked_sub(*s).and_then(|i| p.get(i).copied()) } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } fn inlier_thr(&self) -> f64 { - 4.0 * self.noise + 4.0 * self.noise() } fn huber(&self) -> f64 { - 1.0 * self.noise + self.noise() } } /// Parallax-based keyframes: a new keyframe when the median feature motion since the last -/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared -/// features drop below 24. The first and last frames with features are keyframes. +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. fn keyframes(d: &Data) -> Vec { let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; let mut kf = vec![first]; @@ -97,7 +106,7 @@ fn keyframes(d: &Data) -> Vec { disp.sort_by(f64::total_cmp); let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); let ratio = disp.len() as f64 / base.len().max(1) as f64; - if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { last = j; kf.push(j); } @@ -405,7 +414,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); - let thr = 2.0 * d.noise / f; + let thr = 2.0 * d.noise() / f; let mut cands: Vec = vec![]; if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); @@ -415,7 +424,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } if use_h { let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); - if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { let h = fit.h.0; // Normalised: K⁻¹ H K with K = diag(f, f, 1). let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; @@ -651,6 +660,12 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; let mut results: Vec<(Attempt, f64)> = vec![]; if try_general { let f = match fixed_f { @@ -666,7 +681,20 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo results.push((a, f)); } } - if try_tripod && !cancelled() { + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { let f = match fixed_f { Some(f) => f, None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), @@ -696,7 +724,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if debug() { eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); } - if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } } (Some(g), None) => g, (None, Some(t)) => t, @@ -717,15 +745,44 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (None, true) => FocalMode::PerCamera(F_PRIOR), (None, false) => FocalMode::Shared, }; + lap("tripod"); // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. let kf_frames: Vec = r.cams.keys().copied().collect(); r.adjust(&d, &kf_frames, &cfg, focal, 40); r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } r.triangulate(&d, kind, 1.0, 1.0); r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); if cancelled() { return err("cancelled"); } + lap("keyframe adjustment"); // Every other frame: resect from the points (nearest registered camera first). let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); @@ -746,6 +803,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if cancelled() { return err("cancelled"); } + lap("resection"); // Points for the tracks that only lived between keyframes, then the final adjustment over // every frame (when that stays affordable). r.triangulate(&d, kind, 1.0, 1.0); @@ -786,6 +844,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } (m, _) => m, }; + lap("final adjustment"); Ok(finish(&d, tracks, s, r, method, kind)) } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 258161f98..4b5a83460 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -358,3 +358,81 @@ fn perf_long_clip() { eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); assert!(s.average_error < 0.5); } + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..d581ef35c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -727,6 +727,11 @@ impl EffectcraftApp { self.session.poll_warp(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 000000000..41fb7204f --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f7..11640d88c 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -545,6 +545,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +921,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..ead1fe99a 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b26..364e114e5 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb85..c9dd70dbf 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 000000000..45374fba4 --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356d..bda72461d 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb5..f9f0586fa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec24..25e5a0801 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **258 of 298** After Effects 2026 effects by name (87%); ≈ 70% allowing for simplified implementations | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems: Roto Brush and Refine Edge, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -93,13 +93,13 @@ wasm threads. | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 62% | 18.5 | Advanced 3D (models, PBR, image-based light), multi-view layouts (the 3D Camera Tracker landed in M12.6) | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, rolling-shutter repair, lens distortion in the camera solve (point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing @@ -107,7 +107,7 @@ wasm threads. Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +Keylight (our Screen Key stands in) and Color Profile Converter. Boris FX Mocha and Cineware are third-party and not counted. ## Highest-value gaps, in order