diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index f89c0e87b..d52b19c7c 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/quad.gltf` | EffectCraft contributors | Original work: glTF 2.0 test fixture (textured quad, node hierarchy, translation animation; data URIs) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/quad.glb` | EffectCraft contributors | Original work: binary glTF test fixture (the same quad with its texture in the BIN chunk) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | diff --git a/Cargo.lock b/Cargo.lock index 94e074576..6108b643b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,6 +971,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -1133,6 +1142,10 @@ dependencies = [ "effectcraft-ui-egui", "eframe", "egui", + "log", + "muda", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "rfd", "serde_json", @@ -1145,6 +1158,7 @@ name = "effectcraft-automation" version = "0.1.0" dependencies = [ "effectcraft-engine", + "effectcraft-script", "image", "serde_json", "thiserror 2.0.21", @@ -1184,6 +1198,7 @@ dependencies = [ "effectcraft-raster", "effectcraft-track", "rayon", + "serde", "serde_json", ] @@ -1199,12 +1214,15 @@ dependencies = [ "effectcraft-model", "effectcraft-path", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-text", "effectcraft-time", "effectcraft-track", "image", + "kurbo", "log", "rayon", "serde", @@ -1219,14 +1237,18 @@ version = "0.1.0" dependencies = [ "effectcraft-color", "effectcraft-media", + "effectcraft-opusenc", "effectcraft-project", "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", + "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1288,6 +1310,11 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-psd", + "effectcraft-raster", + "effectcraft-script", + "effectcraft-svg", + "effectcraft-time", "serde_json", ] @@ -1326,8 +1353,10 @@ version = "0.1.0" dependencies = [ "effectcraft-model", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-time", "exr", "filmcraft-codecs", @@ -1353,6 +1382,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-opusenc" +version = "0.1.0" + [[package]] name = "effectcraft-path" version = "0.1.0" @@ -1378,6 +1411,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-psd" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-raster" version = "0.1.0" @@ -1403,11 +1443,39 @@ dependencies = [ "effectcraft-time", "kurbo", "log", + "lz4_flex", "rayon", "serde", "web-time", ] +[[package]] +name = "effectcraft-script" +version = "0.1.0" +dependencies = [ + "boa_engine", + "effectcraft-engine", + "effectcraft-expr", + "effectcraft-keyframe", + "effectcraft-path", + "effectcraft-project", + "effectcraft-render", + "effectcraft-time", + "serde_json", +] + +[[package]] +name = "effectcraft-svg" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "effectcraft-path", + "effectcraft-raster", + "kurbo", + "rayon", + "roxmltree", +] + [[package]] name = "effectcraft-text" version = "0.1.0" @@ -1436,6 +1504,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] @@ -1466,6 +1535,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" @@ -2622,6 +2695,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keyboard-types" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fbe853b403ae61a04233030ae8a79d94975281ed9770a1f9e246732b534b28d" +dependencies = [ + "bitflags 2.13.2", + "serde", +] + [[package]] name = "khronos-egl" version = "6.0.0" @@ -2766,6 +2849,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "mach2" version = "0.4.3" @@ -2829,6 +2918,25 @@ dependencies = [ "pxfm", ] +[[package]] +name = "muda" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6744ab645ebe79f9c428455c7cb9dfa4275fa3b50a25408e08f927da8dc618c" +dependencies = [ + "crossbeam-channel", + "dpi", + "keyboard-types", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "png", + "thiserror 2.0.21", + "windows-sys 0.61.2", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -3987,6 +4095,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index ce22eaac3..22709f60d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,15 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } +effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-opusenc = { path = "crates/opusenc" } +effectcraft-script = { path = "crates/script" } # foundation thiserror = "2" @@ -68,6 +73,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -131,9 +139,15 @@ opt-level = 3 [profile.dev.package.effectcraft-media] opt-level = 3 +[profile.dev.package.effectcraft-opusenc] +opt-level = 3 + [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979cf..2bf7d8ba7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 298 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7, the 3D Camera Tracker in M12.6). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66c..4f2192fc9 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,12 +9,13 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue //! effectcraft-cli bench [--comp C] [--time S] [--scale K] [--n N] [--play N] [--gpu] render timings //! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half) +//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script //! effectcraft-cli mcp [--bridge PORT] MCP server on stdio //! //! Project: --project F.ecproj (or a positional *.ecproj) | --demo | --empty (default: demo; mcp: empty) @@ -32,7 +33,7 @@ use effectcraft_automation::tools::{self, Reply}; use effectcraft_automation::{Backend, McpServer}; use serde_json::{Value, json}; -const USAGE: &str = "usage: effectcraft-cli [args] [--json] +const USAGE: &str = "usage: effectcraft-cli [args] [--json] info project + engine summary commands [--filter TEXT] [--enabled] list engine commands exec [--params JSON] run one engine command @@ -47,6 +48,9 @@ const USAGE: &str = "usage: effectcraft-cli : id or name, '-' = active comp; : id, '#n' or name; : JSON or bare string"; @@ -68,6 +72,7 @@ const VALUED: &[&str] = &[ "--out", "--expression", "--depth", + "--eval", // render "--format", "--start", @@ -243,6 +248,49 @@ fn tool(b: &mut Backend, name: &str, a: Value) -> Result { } } +/// `script FILE.jsx [project]` / `script --eval CODE`: run JavaScript against a headless session +/// (an empty project unless one is given, or `--demo`) or the running app (`--bridge`). +fn script_cmd(args: &Args, json_out: bool) -> Result<(), Failure> { + let (code, name) = match (args.opt("--eval"), args.pos.first()) { + (Some(c), _) => (c.to_string(), "eval".to_string()), + (None, Some(f)) => (std::fs::read_to_string(f).map_err(|e| Failure::Error(format!("cannot read {f}: {e}")))?, f.clone()), + (None, None) => return usage_err("script needs a .jsx/.js file or --eval CODE"), + }; + let mut b = backend(args, false)?; + let r = b.exec("script.run", json!({"code": code, "name": name}))?; + let saved = maybe_save(&mut b, args)?; + if json_out { + let mut o = r.clone(); + if let Some(p) = &saved { + o["saved"] = json!(p); + } + emit(&o, true); + } else { + if let Some(out) = r["output"].as_str().filter(|o| !o.is_empty()) { + println!("{out}"); + } + if !r["result"].is_null() && r["error"].is_null() { + println!("{}", r["result"]); + } + if let Some(p) = saved { + eprintln!("saved {p}"); + } + } + if let Some(e) = r.get("error").filter(|e| !e.is_null()) { + let pos = match (e["line"].as_u64(), e["column"].as_u64()) { + (Some(l), Some(c)) => format!(":{l}:{c}"), + (Some(l), None) => format!(":{l}"), + _ => String::new(), + }; + let msg = format!("{}{pos}: {}", e["file"].as_str().unwrap_or(&name), e["message"].as_str().unwrap_or("script error")); + if json_out { + std::process::exit(1); + } + return Err(Failure::Error(msg)); + } + Ok(()) +} + /// Print a result: compact JSON with `--json`, else pretty JSON. fn emit(v: &Value, json_out: bool) { if json_out { @@ -315,6 +363,7 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> { } } } + "script" => script_cmd(args, json_out)?, "exec" => { let Some(id) = args.pos.first().cloned() else { return usage_err("exec needs a command id") }; let params = match (args.opt("--params"), args.pos.get(1).map(String::as_str)) { diff --git a/apps/effectcraft-cli/tests/cli.rs b/apps/effectcraft-cli/tests/cli.rs index c84277202..d7c48762f 100644 --- a/apps/effectcraft-cli/tests/cli.rs +++ b/apps/effectcraft-cli/tests/cli.rs @@ -131,3 +131,34 @@ fn mcp_over_stdio() { assert_eq!(replies[2]["result"]["isError"], false, "{}", replies[2]); assert_eq!(replies[3]["result"]["content"][0]["type"], "image"); } + +#[test] +fn script_file_and_eval() { + let proj = tmp("scripted.ecproj"); + let p = proj.to_str().unwrap(); + let jsx = tmp("build.jsx"); + std::fs::write( + &jsx, + "app.beginUndoGroup('Build');\nvar c = app.project.items.addComp('FromJsx', 160, 90, 1, 2, 24);\nc.layers.addSolid([1, 0, 0], 'Red', 160, 90, 1);\napp.endUndoGroup();\nwriteLn('built ' + c.name);\nc.numLayers", + ) + .unwrap(); + let v = ok_json(&["script", jsx.to_str().unwrap(), "--save-as", p]); + assert_eq!(v["ok"], json!(true), "{v}"); + assert_eq!(v["result"], json!(1)); + assert_eq!(v["output"], json!("built FromJsx")); + assert_eq!(v["saved"], p); + // Run against the saved project (positional), plain output. + let out = bin().args(["script", "--eval", "app.project.item(1).name + ' has ' + app.project.item(1).numLayers", p]).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "\"FromJsx has 1\""); + // Errors: exit 1 with file:line:col. + let bad = tmp("bad.jsx"); + std::fs::write(&bad, "var a = 1;\n\nmissingFunction();\n").unwrap(); + let out = bin().args(["script", bad.to_str().unwrap()]).output().unwrap(); + assert_eq!(out.status.code(), Some(1)); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains("bad.jsx:3:1") && err.contains("missingFunction"), "{err}"); + let (code, v) = run_json(&["script", "--eval", "\nnope()"]); + assert_eq!(code, 1); + assert_eq!(v["error"]["line"], json!(2)); +} diff --git a/apps/effectcraft-web/src/lib.rs b/apps/effectcraft-web/src/lib.rs index 7a765bfc6..edd90e311 100644 --- a/apps/effectcraft-web/src/lib.rs +++ b/apps/effectcraft-web/src/lib.rs @@ -105,8 +105,8 @@ fn query_flag(name: &str) -> bool { web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()).is_some_and(|p| p.has(name)) } -/// A session wired for the browser: media decoding from memory, expressions, Render Queue export -/// to downloads. +/// A session wired for the browser: media decoding from memory, expressions, scripting, Render +/// Queue export to downloads. pub fn session() -> Session { let pool = Arc::new(effectcraft_media::MediaPool::new()); Session { @@ -115,6 +115,7 @@ pub fn session() -> Session { importer: Some(Arc::new(files::WebImporter { pool })), expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_host::script::runner), exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), ..Default::default() } diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 6b34763f2..3d5e1bb12 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -18,6 +18,7 @@ eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", " serde_json = { workspace = true } rfd = { workspace = true } cpal = { workspace = true } +log = { workspace = true } [build-dependencies] # Icon + VERSIONINFO for effectcraft.exe; build.rs only uses it when targeting Windows. @@ -28,3 +29,8 @@ winresource = "0.1" winit = { version = "0.30", default-features = false } # Keep the control channel responsive when the window is covered (App Nap / occlusion throttling). objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSProcessInfo", "NSString", "NSObject"] } +# The native menu bar (NSMenu). MIT OR Apache-2.0; macOS only, never in the web build. +muda = { version = "0.21", default-features = false } +objc2 = "0.6" +# NSApplication hide / hide others / show all, and the pasteboard for Edit ▸ Paste in text fields. +objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSApplication", "NSResponder", "NSPasteboard"] } diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 34b40a84d..b6426f5f0 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -7,6 +7,8 @@ mod audio_out; mod control_server; +#[cfg(target_os = "macos")] +mod native_menu; use effectcraft_ui_egui::EffectcraftApp; use serde_json::json; @@ -100,16 +102,52 @@ fn main() -> eframe::Result { app.hooks.pick_save_file = Some(Box::new(|name: &str, ext: &str| { rfd::FileDialog::new().add_filter(ext, &[ext]).set_file_name(name).save_file().map(|p| p.to_string_lossy().to_string()) })); + #[cfg(target_os = "macos")] + { + app.hooks.app_action = Some(Box::new(native_menu::app_action)); + app.hooks.clipboard_text = Some(Box::new(native_menu::clipboard_text)); + } if let Some(port) = control_port { disable_app_nap(); let rx = control_server::start(port, cc.egui_ctx.clone()); app = app.with_control(rx); } - Ok(Box::new(app)) + Ok(Box::new(Desktop { + #[cfg(target_os = "macos")] + menu: native_menu::NativeBar::new(&cc.egui_ctx), + app, + })) }), ) } +/// The desktop app: the egui app plus, on macOS, the system menu bar. +struct Desktop { + app: EffectcraftApp, + #[cfg(target_os = "macos")] + menu: native_menu::NativeBar, +} + +impl eframe::App for Desktop { + fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { + #[cfg(target_os = "macos")] + self.menu.update(&mut self.app, ctx); + self.app.logic(ctx, frame); + } + + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { + self.app.ui(ui, frame); + } + + fn on_exit(&mut self) { + self.app.on_exit(); + } + + fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { + self.app.raw_input_hook(ctx, raw_input); + } +} + /// The platform config directory for EffectCraft (`EFFECTCRAFT_CONFIG_DIR` overrides): /// `~/Library/Application Support/EffectCraft` (macOS), `%APPDATA%\EffectCraft` (Windows), /// `$XDG_CONFIG_HOME/effectcraft` or `~/.config/effectcraft` (Linux and others). diff --git a/apps/effectcraft/src/native_menu.rs b/apps/effectcraft/src/native_menu.rs new file mode 100644 index 000000000..a20b27f5f --- /dev/null +++ b/apps/effectcraft/src/native_menu.rs @@ -0,0 +1,210 @@ +//! macOS: the menu bar in the system menu bar (`NSMenu` through `muda`), built from +//! [`effectcraft_ui_egui::native_menu`]'s spec of the engine's After Effects menu tree. +//! +//! Each frame [`NativeBar::update`] runs menu activations (the same command ids as the in-window +//! bar), and when [`native_menu::state_key`] changed it rebuilds the spec and pushes labels, +//! enabled and check state into the existing items; a structural change (Open Recent, a new +//! shortcut preset) rebuilds the menu. Settings ▸ Appearance ▸ Use In-Window Menu Bar on macOS +//! swaps back to the in-window bar. + +use std::collections::HashMap; +use std::sync::mpsc::{Receiver, channel}; + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::native_menu::{self, NativeMenu, NativeNode, Role}; +use muda::{CheckMenuItem, Menu, MenuEvent, PredefinedMenuItem, Submenu}; + +/// The `muda` accelerator for a spec accelerator string. +pub fn parse_accelerator(s: &str) -> Option { + s.parse().ok() +} + +pub struct NativeBar { + menu: Option, + items: HashMap, + spec: NativeMenu, + structure: u64, + state: Option, + events: Receiver, + /// The in-window-bar setting last applied (None: not yet). + in_window: Option, +} + +impl NativeBar { + pub fn new(ctx: &egui::Context) -> Self { + let (tx, rx) = channel(); + let ctx = ctx.clone(); + MenuEvent::set_event_handler(Some(move |e: MenuEvent| { + let _ = tx.send(e); + ctx.request_repaint(); + })); + NativeBar { menu: None, items: HashMap::new(), spec: NativeMenu::default(), structure: 0, state: None, events: rx, in_window: None } + } + + /// Run activations and keep the native menu in step with the app (call once per frame). + pub fn update(&mut self, app: &mut EffectcraftApp, ctx: &egui::Context) { + let in_window = app.session.prefs.appearance.in_window_menu_bar_mac; + if self.in_window != Some(in_window) { + self.in_window = Some(in_window); + app.ui.show_menu_bar = in_window; + if in_window { + if let Some(m) = self.menu.take() { + m.remove_for_nsapp(); + } + self.items.clear(); + } else { + self.state = None; + self.structure = 0; + } + } + while let Ok(e) = self.events.try_recv() { + if let Err(err) = native_menu::activate(app, ctx, &self.spec, &e.id.0) { + log::warn!("menu: {err}"); + } + // Check items toggle themselves when clicked: resync with the app. + self.state = None; + } + if in_window { + return; + } + let key = native_menu::state_key(app); + if self.state == Some(key) && self.menu.is_some() { + return; + } + self.state = Some(key); + let spec = native_menu::build(app); + let structure = spec.structure(); + if self.menu.is_none() || structure != self.structure { + self.install(&spec); + self.structure = structure; + } else { + self.refresh(&spec); + } + self.spec = spec; + } + + fn install(&mut self, spec: &NativeMenu) { + if let Some(m) = self.menu.take() { + m.remove_for_nsapp(); + } + self.items.clear(); + let menu = Menu::new(); + for node in &spec.menus { + if let NativeNode::Submenu { label, children } = node { + let sub = Submenu::new(label, true); + self.fill(&sub, children); + match label.as_str() { + "Window" => sub.set_as_windows_menu_for_nsapp(), + "Help" => sub.set_as_help_menu_for_nsapp(), + _ => {} + } + if let Err(e) = menu.append(&sub) { + log::warn!("menu {label}: {e}"); + } + } + } + menu.init_for_nsapp(); + self.menu = Some(menu); + } + + fn fill(&mut self, parent: &Submenu, nodes: &[NativeNode]) { + for n in nodes { + let r = match n { + NativeNode::Separator => parent.append(&PredefinedMenuItem::separator()), + NativeNode::Predefined { role, label, .. } => { + let item = match role { + Role::Services => PredefinedMenuItem::services(Some(label)), + Role::Hide => PredefinedMenuItem::hide(Some(label)), + Role::HideOthers => PredefinedMenuItem::hide_others(Some(label)), + Role::ShowAll => PredefinedMenuItem::show_all(Some(label)), + }; + parent.append(&item) + } + NativeNode::Submenu { label, children } => { + let sub = Submenu::new(label, true); + self.fill(&sub, children); + parent.append(&sub) + } + NativeNode::Item(i) => { + let accel = i.accelerator.as_deref().and_then(parse_accelerator); + let item = CheckMenuItem::with_id(i.id.clone(), &i.label, i.enabled, i.checked == Some(true), accel); + let r = parent.append(&item); + self.items.insert(i.id.clone(), item); + r + } + }; + if let Err(e) = r { + log::warn!("menu item: {e}"); + } + } + } + + /// Push label / enabled / check changes into the existing items. + fn refresh(&self, spec: &NativeMenu) { + let old: HashMap<&str, &native_menu::NativeItem> = self.spec.items().into_iter().map(|i| (i.id.as_str(), i)).collect(); + for i in spec.items() { + let Some(native) = self.items.get(&i.id) else { continue }; + let prev = old.get(i.id.as_str()); + if prev.is_none_or(|p| p.label != i.label) { + native.set_text(&i.label); + } + if prev.is_none_or(|p| p.enabled != i.enabled) { + native.set_enabled(i.enabled); + } + let checked = i.checked == Some(true); + if native.is_checked() != checked { + native.set_checked(checked); + } + } + } +} + +/// `app.hide` / `app.hideOthers` / `app.showAll` through NSApplication (main thread only). +pub fn app_action(id: &str) -> bool { + use objc2::MainThreadMarker; + use objc2_app_kit::NSApplication; + let Some(mtm) = MainThreadMarker::new() else { return false }; + let app = NSApplication::sharedApplication(mtm); + match id { + "app.hide" => app.hide(None), + "app.hideOthers" => app.hideOtherApplications(None), + "app.showAll" => app.unhideAllApplications(None), + _ => return false, + } + true +} + +/// The general pasteboard's text. +pub fn clipboard_text() -> Option { + use objc2_app_kit::NSPasteboard; + // NSPasteboardTypeString's value (the UTI), without touching the extern static. + let ty = objc2_foundation::NSString::from_str("public.utf8-plain-text"); + NSPasteboard::generalPasteboard().stringForType(&ty).map(|s| s.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn every_accelerator_parses_with_muda() { + let mut s = effectcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let app = EffectcraftApp::new(s); + let spec = native_menu::build(&app); + let mut n = 0; + for i in spec.items() { + if let Some(a) = &i.accelerator { + assert!(parse_accelerator(a).is_some(), "{} `{a}` does not parse", i.label); + n += 1; + } + } + assert!(n > 50, "only {n} accelerators"); + // Spot checks against the AE shortcuts. + let accel = |cmd: &str| spec.items().into_iter().find(|i| i.command == cmd).and_then(|i| i.accelerator.clone()); + assert_eq!(accel("file.saveAs").as_deref(), Some("Cmd+Shift+S")); + assert_eq!(accel("comp.new").as_deref(), Some("Cmd+N")); + assert_eq!(parse_accelerator("Cmd+Shift+S"), "CmdOrCtrl+Shift+S".parse().ok()); + } +} diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index a3b164080..0b2bcb051 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -14,3 +14,6 @@ effectcraft-engine = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } image = { workspace = true } + +[dev-dependencies] +effectcraft-script = { workspace = true } diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index f175b12f9..848d3ee0b 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -54,6 +54,7 @@ fn initialize_and_list_tools() { for want in [ "list_commands", "execute_command", + "run_script", "get_project", "get_comp", "get_layer", @@ -263,3 +264,30 @@ fn bridge_rejects_non_loopback() { assert!(Backend::bridge("10.0.0.1:9877").is_err()); assert!(Backend::bridge("localhost:9877").is_ok()); } + +#[test] +fn run_script_round_trip() { + let mut sess = Session::default(); + effectcraft_script::install(&mut sess); + let mut s = McpServer::new(Backend::headless(sess)); + let r = call_json( + &mut s, + "run_script", + json!({"code": "var c = app.project.items.addComp('Scripted', 320, 180, 1, 2, 24);\nc.layers.addText('Hi');\nwriteLn('made ' + c.name);\nc.numLayers"}), + ); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"], json!(1)); + assert_eq!(r["output"], json!("made Scripted")); + // The edit is visible to the other tools. + let p = call_json(&mut s, "get_project", json!({})); + assert!(p.to_string().contains("Scripted"), "{p}"); + // Errors come back with their line. + let r = call_json(&mut s, "run_script", json!({"code": "var a = 1;\nundefinedFn();", "name": "bad.jsx"})); + assert_eq!(r["ok"], json!(false)); + assert_eq!(r["error"]["line"], json!(2)); + assert_eq!(r["error"]["file"], json!("bad.jsx")); + // Without a scripting engine the command explains itself. + let mut bare = server(); + let (c, err) = call(&mut bare, "run_script", json!({"code": "1"})); + assert!(err && c[0]["text"].as_str().unwrap().contains("not available"), "{c:?}"); +} diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 205573b65..739c5524b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -140,6 +140,15 @@ fn execute_command(b: &mut Backend, a: &Value) -> Result { json_reply(b.exec(id, params)?) } +fn run_script(b: &mut Backend, a: &Value) -> Result { + let code = get(a, "code").and_then(Value::as_str).ok_or_else(|| Error::BadArgs("missing `code` (JavaScript)".into()))?; + let mut p = json!({"code": code}); + if let Some(n) = get(a, "name") { + p["name"] = n.clone(); + } + json_reply(b.exec("script.run", p)?) +} + fn get_project(b: &mut Backend, _: &Value) -> Result { json_reply(b.exec("project.summary", json!({}))?) } @@ -386,6 +395,21 @@ static TOOLS: &[ToolDef] = &[ }, run: execute_command, }, + ToolDef { + name: "run_script", + description: "Run JavaScript with an After Effects-style scripting object model (app, app.project, items.addComp, comp.layers.addSolid/addText/addShape/addNull/addCamera/addLight, layer.property(\"ADBE Transform Group\").property(\"ADBE Position\").setValueAtTime(t, v), Effects.addProperty(\"ADBE Gaussian Blur 2\"), expressions, markers, renderQueue…). Every edit is an undoable engine command; app.beginUndoGroup/endUndoGroup make one undo step. Returns {ok, result (last expression value), output (writeLn/$.writeln/alert text), error: {message, line, column} | null}. File access is limited to the project folder unless the user allows scripts to write files.", + bridge_only: false, + schema: || { + schema( + json!({ + "code": {"type": "string", "description": "JavaScript source, e.g. `var c = app.project.items.addComp(\"Main\", 1920, 1080, 1, 5, 30); c.layers.addText(\"Hi\"); c.numLayers`."}, + "name": {"type": "string", "description": "Script name used in error messages (default `script`)."} + }), + &["code"], + ) + }, + run: run_script, + }, ToolDef { name: "get_project", description: "Project overview: file path, dirty flag, bit depth, every item (id, name, type Composition/Footage/Solid/Folder, size, duration, layer count), the active comp id, current time (s), selected layers and the undo stack.", diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index ff5d3ed75..965cdbd22 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -17,5 +17,6 @@ effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-track = { workspace = true } rayon = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 000000000..dd97b585e --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 46da720e3..6f47b01e9 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio_fx; mod blur2; mod blur3; +pub mod camera_tracker; mod channel; mod channel2; mod channel3d; @@ -37,6 +38,7 @@ pub mod paint; mod perspective; mod perspective2; pub mod puppet; +pub mod roto; mod sim; mod sim2; mod sim3; @@ -381,6 +383,8 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); + v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -456,7 +460,7 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS.iter().chain(roto::GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -528,6 +532,10 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.obsolete.mochashape", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, + // Each frame has its own segmentation. + roto::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs new file mode 100644 index 000000000..a1f93a923 --- /dev/null +++ b/crates/effects/src/roto.rs @@ -0,0 +1,698 @@ +//! Roto Brush & Refine Edge (applied by the Roto Brush tool; Effect ▸ Matte). +//! +//! The instance stores the user's strokes, base frame and segmentation span as JSON in its hidden +//! **Strokes** parameter ([`RotoData`]); the segmentation itself is *derived* data: each frame's +//! result is a deterministic function of its chain key ([`RotoData::chain_keys`] seeded with +//! [`seed`]: what the frames are, the Roto Brush settings and the strokes from the base frame +//! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the +//! engine's propagation job fills at full resolution and rendering fills on demand (reading +//! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project +//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen** +//! parameter ([`FrozenCache`]); they are used while their key still matches. +//! +//! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather, +//! Contrast) → Refine Edge matting inside the Refine Edge band (guided filter + closed-form +//! matting with Higher Quality) → Refine Edge Matte adjustments → matte motion blur → +//! decontamination of the edge colours. Outside the segmentation span the layer is transparent; +//! without strokes the effect passes its input through. +//! +//! *Version* is stored but all versions use the same (classic, graph-cut) engine; *Quality ▸ +//! Best* segments at a higher working resolution with more iterations. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_track::roto::refine::{self, MatteParams}; +use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle}; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +pub use effectcraft_track::roto::{Stroke, StrokeKind}; + +/// The effect's id. +pub const ID: &str = "ec.matte.rotobrush"; +/// Hidden parameters. +pub const STROKES: &str = "strokes"; +pub const INPUT_KEY: &str = "inputKey"; +pub const FROZEN: &str = "frozen"; + +/// Default segmentation span each side of the base frame (frames). +pub const DEFAULT_SPAN: i64 = 20; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("rotoBrushMatte", "Roto Brush Matte"), ("refineEdgeMatte", "Refine Edge Matte")]; + +pub const VERSIONS: [&str; 3] = ["1.0", "2.0", "3.0"]; +pub const QUALITIES: [&str; 2] = ["Standard", "Best"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Roto Brush & Refine Edge", + category: "Matte", + params: vec![ + p("version", "Version", Value::Enum(2), popup(&VERSIONS)), + p("quality", "Quality", Value::Enum(0), popup(&QUALITIES)), + p("rotoBrushMatte/searchRadius", "Search Radius", num(15.0), slider(0.0, 100.0, 0.0, 50.0, 1)), + p("rotoBrushMatte/motionThreshold", "Motion Threshold", num(10.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/motionDamping", "Motion Damping", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/viewSearchRegion", "View Search Region", Value::Bool(false), ParamUi::Checkbox), + p("rotoBrushMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)), + p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("refineEdgeMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/useMotionBlur", "Use Motion Blur", Value::Bool(false), ParamUi::Checkbox), + p("refineEdgeMatte/motionBlurSamples", "Motion Blur Samples", num(11.0), slider(1.0, 64.0, 1.0, 32.0, 0)), + p("refineEdgeMatte/shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 0)), + p("refineEdgeMatte/higherQuality", "Higher Quality", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminateEdgeColors", "Decontaminate Edge Colors", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminationAmount", "Decontamination Amount", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/extendWhereSmoothed", "Extend Where Smoothed", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/increaseDecontaminationRadius", "Increase Decontamination Radius", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/viewDecontaminationMap", "View Decontamination Map", Value::Bool(false), ParamUi::Checkbox), + p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// The instance's strokes (parsed once per content). +pub fn data(params: &Params) -> Arc { + static C: OnceLock>>> = OnceLock::new(); + let j = s(params, STROKES); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return d.clone(); + } + let d = Arc::new(RotoData::from_json(j)); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 64 { + m.clear(); + } + m.insert(h, d.clone()); + d +} + +/// Segmentation settings at `scale` (pixels per layer pixel). +pub fn seg_opts(params: &Params, scale: f64) -> SegOpts { + SegOpts { + scale, + search_radius: f(params, "rotoBrushMatte/searchRadius").max(0.0), + motion_threshold: f(params, "rotoBrushMatte/motionThreshold"), + motion_damping: f(params, "rotoBrushMatte/motionDamping"), + best: e(params, "quality") == 1, + } +} + +/// Matte settings. +pub fn matte_params(params: &Params) -> MatteParams { + MatteParams { + rb_feather: f(params, "rotoBrushMatte/feather").max(0.0), + rb_contrast: f(params, "rotoBrushMatte/contrast"), + rb_shift: f(params, "rotoBrushMatte/shiftEdge"), + rb_chatter: f(params, "rotoBrushMatte/reduceChatter"), + smooth: f(params, "refineEdgeMatte/smooth").max(0.0), + feather: f(params, "refineEdgeMatte/feather").max(0.0), + contrast: f(params, "refineEdgeMatte/contrast"), + shift: f(params, "refineEdgeMatte/shiftEdge"), + chatter: f(params, "refineEdgeMatte/reduceChatter"), + motion_blur: b(params, "refineEdgeMatte/useMotionBlur"), + mb_samples: f(params, "refineEdgeMatte/motionBlurSamples").round().clamp(1.0, 64.0) as usize, + shutter_angle: f(params, "refineEdgeMatte/shutterAngle"), + higher_quality: b(params, "refineEdgeMatte/higherQuality"), + decontaminate: b(params, "refineEdgeMatte/decontaminateEdgeColors"), + decontamination: f(params, "refineEdgeMatte/decontaminationAmount"), + extend_where_smoothed: b(params, "refineEdgeMatte/extendWhereSmoothed"), + increase_radius: f(params, "refineEdgeMatte/increaseDecontaminationRadius").max(0.0), + } +} + +/// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer +/// size, the frame rate and the segmentation settings. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [ + layer_size[0], + layer_size[1], + fps, + f(params, "rotoBrushMatte/searchRadius"), + f(params, "rotoBrushMatte/motionThreshold"), + f(params, "rotoBrushMatte/motionDamping"), + ] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]); + h +} + +/// The layer frame shown at layer time `t` seconds. +pub fn frame_of(t: f64, fps: f64) -> i64 { + (t * fps + 1e-6).floor() as i64 +} + +// ------------------------------------------------------------------ derived-data cache + +struct SegCache { + map: HashMap<(u64, u64), Arc>, + order: VecDeque<(u64, u64)>, + bytes: usize, +} + +const CACHE_BYTES: usize = 768 << 20; + +fn cache() -> &'static Mutex { + static C: OnceLock> = OnceLock::new(); + C.get_or_init(|| Mutex::new(SegCache { map: HashMap::new(), order: VecDeque::new(), bytes: 0 })) +} + +/// A cached segmentation (chain key, scale). +pub fn cached(key: u64, scale: f64) -> Option> { + cache().lock().unwrap_or_else(|e| e.into_inner()).map.get(&(key, scale.to_bits())).cloned() +} + +pub fn store(key: u64, scale: f64, seg: Arc) { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let k = (key, scale.to_bits()); + if c.map.contains_key(&k) { + return; + } + c.bytes += seg.matte.len() + seg.refine.len(); + c.map.insert(k, seg); + c.order.push_back(k); + while c.bytes > CACHE_BYTES { + let Some(old) = c.order.pop_front() else { break }; + if let Some(s) = c.map.remove(&old) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } +} + +/// Drop the cached segmentations of these chain keys (every scale). +pub fn forget(keys: impl IntoIterator) { + let keys: std::collections::HashSet = keys.into_iter().collect(); + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let gone: Vec<(u64, u64)> = c.map.keys().filter(|k| keys.contains(&k.0)).copied().collect(); + for k in gone { + if let Some(s) = c.map.remove(&k) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } + c.order.retain(|k| !keys.contains(&k.0)); +} + +/// Drop every cached segmentation (Edit ▸ Purge). +pub fn purge() { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + c.map.clear(); + c.order.clear(); + c.bytes = 0; +} + +/// The layer-space grid (`round(size · scale)` pixels) of a buffer. +pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image { + let s = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1)); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = buf.img.get(x as i64 + ox, y as i64 + oy); + } + }); + img +} + +/// Everything needed to compute an instance's segmentations. +pub struct Chain { + pub data: Arc, + pub keys: BTreeMap, + pub opts: SegOpts, + pub refine_radius: f64, +} + +impl Chain { + pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain { + let data = data(params); + let keys = data.chain_keys(seed(params, layer_size, fps)); + Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale) } + } + + /// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest + /// cached frame towards the base (full-resolution results serve smaller scales). + /// `frame(g)`: the effect's input on frame g on the layer grid at this scale. + pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool) -> Option> { + let sc = self.opts.scale; + let lookup = |g: i64| -> Option> { + let k = *self.keys.get(&g)?; + if let Some(s) = cached(k, sc) { + return Some(s); + } + if sc != 1.0 { + let full = cached(k, 1.0)?; + let s = Arc::new(downscale(&full, sc)); + store(k, sc, s.clone()); + return Some(s); + } + None + }; + if let Some(s) = lookup(f) { + return Some(s); + } + let base = self.data.base?; + if !self.data.in_span(f) { + return None; + } + // Walk towards the base to the nearest available frame. + let mut path = vec![f]; + let mut g = f; + let mut start: Option> = None; + while let Some(src) = self.data.source_of(g) { + if let Some(s) = lookup(src) { + start = Some(s); + break; + } + path.push(src); + g = src; + } + path.reverse(); + let mut prev_img: Option> = None; + let mut cur = start; + for g in path { + if cancel() { + return None; + } + let img = frame(g)?; + let strokes = self.data.strokes_at(g); + let seg = match (&cur, g == base) { + (_, true) | (None, _) => rb::segment(&img, &strokes, None, &self.opts, self.refine_radius), + (Some(p), false) => { + let src = self.data.source_of(g)?; + let pimg = match prev_img.take() { + Some(i) => i, + None => frame(src)?, + }; + if pimg.width != img.width || pimg.height != img.height { + return None; + } + rb::propagate(&pimg, p, &img, &strokes, &self.opts, self.refine_radius) + } + }; + let seg = Arc::new(seg); + store(*self.keys.get(&g)?, sc, seg.clone()); + cur = Some(seg); + prev_img = Some(img); + } + cur + } + + /// Whether frame `f` is computed (at full resolution or at `scale`). + pub fn is_cached(&self, f: i64, scale: f64) -> bool { + self.keys.get(&f).is_some_and(|k| cached(*k, 1.0).is_some() || cached(*k, scale).is_some()) + } +} + +/// Nearest-neighbour resampling of a segmentation to another scale. +pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { + let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize); + resample(s, w, h) +} + +fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { + if s.w == w && s.h == h { + return s.clone(); + } + let mut out = FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], fg: s.fg.clone(), bg: s.bg.clone() }; + if s.w == 0 || s.h == 0 { + return out; + } + for y in 0..h { + let sy = (((y as f64 + 0.5) * s.h as f64 / h as f64) as usize).min(s.h - 1); + for x in 0..w { + let sx = (((x as f64 + 0.5) * s.w as f64 / w as f64) as usize).min(s.w - 1); + out.matte[y * w + x] = s.matte[sy * s.w + sx]; + out.refine[y * w + x] = s.refine.get(sy * s.w + sx).copied().unwrap_or(0); + } + } + out +} + +/// The final matte of frame `seg` (with its neighbours), on its own grid. +pub fn final_alpha( + img: &Image, + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + mp: &MatteParams, + refine_radius: f64, + scale: f64, +) -> Vec { + let rgb = rb::rgb_of(img); + refine::matte_alpha(&rgb, seg, prev, next, mp, refine_radius * scale, scale) +} + +fn needs_neighbours(mp: &MatteParams) -> bool { + mp.rb_chatter > 0.0 || mp.chatter > 0.0 || mp.motion_blur +} + +// ------------------------------------------------------------------ Freeze + +/// Frozen mattes (Freeze): the final 8-bit alpha and the Refine Edge band of every frame of +/// the span at full resolution, valid while `key` matches [`frozen_key`]. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FrozenCache { + pub key: String, + pub size: [usize; 2], + pub frames: BTreeMap, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FrozenFrame { + /// Alpha (RLE, base64). + pub a: String, + /// Refine Edge band (RLE, base64). + pub b: String, +} + +impl FrozenCache { + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } +} + +/// What frozen mattes depend on: every chain key and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String { + let ch = Chain::new(params, layer_size, fps, 1.0); + let mut h = 0xcbf2_9ce4_8422_2325; + for (f, k) in &ch.keys { + fnv(&mut h, &f.to_le_bytes()); + fnv(&mut h, &k.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +type FrozenParsed = Mutex>>; + +fn frozen(params: &Params) -> Option> { + static C: OnceLock = OnceLock::new(); + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + +/// Encode one frame for [`FrozenCache`]. +pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame { + let a: Vec = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect(); + FrozenFrame { a: rle::encode(&a), b: rle::encode(band) } +} + +// ------------------------------------------------------------------ render + +/// A frame's matte (alpha and band) on the layer grid of `img` at `scale`. +fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc, scale: f64) -> Option<(Vec, Vec, f64)> { + let (w, h) = (img.width as usize, img.height as usize); + // Frozen mattes. + if let Some(fc) = frozen(ctx.params) + && fc.key == frozen_key(ctx.params, ctx.layer_size, ctx.fps()) + && let Some(fr) = fc.frames.get(&f) + { + let a = rle::decode(&fr.a)?; + let bnd = rle::decode(&fr.b)?; + let [fw, fh] = fc.size; + if a.len() == fw * fh && bnd.len() == fw * fh { + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + let (alpha, band) = if fw == w && fh == h { + (af, bnd) + } else { + let mut al = vec![0.0f32; w * h]; + let mut bd = vec![0u8; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + al[y * w + x] = rb::matting::sample(&af, fw, fh, sx, sy); + bd[y * w + x] = bnd[(sy as usize).min(fh - 1) * fw + (sx as usize).min(fw - 1)]; + } + } + (al, bd) + }; + let rr = data(ctx.params).refine_radius(); + return Some((alpha, band, rr)); + } + } + let host = ctx.env.host; + let chain = Chain::new(ctx.params, ctx.layer_size, ctx.fps(), scale); + let fps = ctx.fps(); + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let buf = host?.self_at(g as f64 / fps, effects)?; + let im = layer_grid(&buf, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let seg = chain.ensure(f, &mut frame, &|| false)?; + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { resample(&seg, w, h) }; + let mp = matte_params(ctx.params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + for (slot, g) in [(&mut prev, f - 1), (&mut next, f + 1)] { + if chain.data.in_span(g) + && let Some(s) = chain.ensure(g, &mut frame, &|| false) + { + *slot = Some(if s.w == w && s.h == h { (*s).clone() } else { resample(&s, w, h) }); + } + } + } + let alpha = final_alpha(img, &seg, prev.as_ref(), next.as_ref(), &mp, chain.refine_radius, scale); + Some((alpha, seg.refine, chain.refine_radius)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + let d = data(ctx.params); + if d.is_empty() || ctx.adjustment { + return buf; + } + let f = frame_of(ctx.time, ctx.fps()); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + if !d.in_span(f) { + return out; + } + let img = Arc::new(layer_grid(&buf, ctx.layer_size)); + let Some((alpha, band, rr)) = matte_for(ctx, f, &img, sc) else { return out }; + let (w, h) = (img.width as usize, img.height as usize); + let mp = matte_params(ctx.params); + let mut rgb = rb::rgb_of(&img); + let map = refine::decontaminate(&mut rgb, &alpha, &band, w, h, &mp, rr * sc, sc); + let view_map = b(ctx.params, "refineEdgeMatte/viewDecontaminationMap"); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + for y in 0..h { + for x in 0..w { + let (bx, by) = (x as i64 + ox, y as i64 + oy); + if bx < 0 || by < 0 || bx >= out.img.width as i64 || by >= out.img.height as i64 { + continue; + } + let i = y * w + x; + let a_in = img.data[i][3]; + let px = if view_map { + let v = map[i]; + [v, v, v, 1.0] + } else { + let a = alpha[i] * a_in; + [rgb[i][0] * a, rgb[i][1] * a, rgb[i][2] * a, a] + }; + out.img.set(bx as u32, by as u32, px); + } + } + out +} + +/// The final matte of the effect at layer frame `f` on the full-resolution layer grid of +/// `input` (the effect's input buffer at scale 1), computing what is missing. For Freeze and +/// the Layer panel overlays. +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option<(Vec, Vec)> { + let chain = Chain::new(params, layer_size, fps, 1.0); + let img = frame(f)?; + let seg = chain.ensure(f, frame, cancel)?; + let mp = matte_params(params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + if chain.data.in_span(f - 1) { + prev = chain.ensure(f - 1, frame, cancel); + } + if chain.data.in_span(f + 1) { + next = chain.ensure(f + 1, frame, cancel); + } + } + let alpha = final_alpha(&img, &seg, prev.as_deref(), next.as_deref(), &mp, chain.refine_radius, 1.0); + Some((alpha, seg.refine.clone())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A host serving a moving red disk over a blue/green background as the effect's input. + struct Seq; + fn frame(t: f64) -> Image { + let fi = (t * 25.0 + 1e-6).floor(); + let c = [40.0 + 2.0 * fi, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let fg = [0.85 + tex, 0.2, 0.1]; + let bg = [0.1, 0.4 + tex, 0.7]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params_with(d: &RotoData) -> Params { + let s = &specs()[0]; + let mut params = Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() }; + params.values.insert(STROKES.into(), Value::Str(d.to_json())); + params + } + + #[test] + fn renders_the_segmented_disk_on_demand() { + let mut d = RotoData::default(); + d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]] }, 20, [0, 24]); + d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]] }, 20, [0, 24]); + let params = params_with(&d); + let host = Seq; + let run = |t: f64| { + let ctx = EffectCtx { + params: ¶ms, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }).img + }; + // Frame 5 (propagated through frames 1–4). + let out = run(5.0 / 25.0 + 0.001); + let c = [50.0, 40.0]; + let mut wrong = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + wrong += 1; + } + } + } + assert!(wrong < 20, "{wrong} wrong pixels"); + // Outside the span: transparent. + let mut d2 = (*data(¶ms)).clone(); + d2.span = [0, 3]; + let p2 = params_with(&d2); + let ctx = EffectCtx { + params: &p2, + time: 0.3, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + let o = crate::apply(&specs()[0], &ctx, Buf { img: frame(0.3), offset: [0.0; 2], scale: 1.0 }).img; + assert!(o.data.iter().all(|p| p[3] == 0.0)); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index a60242bb4..45508a716 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -23,6 +23,9 @@ effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } effectcraft-model = { workspace = true } effectcraft-lottie = { workspace = true } +effectcraft-psd = { workspace = true } +kurbo = { workspace = true } +effectcraft-svg = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 000000000..1b984aac2 --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 000000000..5d454aed5 --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/create.rs b/crates/engine/src/commands/create.rs new file mode 100644 index 000000000..72cb2420f --- /dev/null +++ b/crates/engine/src/commands/create.rs @@ -0,0 +1,139 @@ +//! Layer ▸ Create: Convert to Editable Text (Photoshop type layers), Create Shapes from Text, +//! Create Masks from Text and Create Shapes from Vector Layer (SVG footage). + +use effectcraft_color::Label; +use effectcraft_project::{ItemKind, LayerSource, Solid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, has_layers, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const OPS: &str = "editableText|shapesFromText|masksFromText|shapesFromVector"; + +fn create(s: &mut Session, p: &Value) -> Result { + let op = str_p(p, "op").ok_or_else(|| bad("layer.create", format!("missing `op` ({OPS})")))?; + let (cid, lids) = layers_p(s, p)?; + if lids.is_empty() { + return Err(bad("layer.create", "select a layer first")); + } + let time = s.time(); + let mut created = vec![]; + let mut skipped = vec![]; + for lid in lids { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let Some(src) = comp.layer(lid).cloned() else { continue }; + let footage = src.source.item().and_then(|i| match &s.project.item(i)?.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }); + match op { + "shapesFromVector" => { + let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else { + skipped.push(format!("{}: not a vector (SVG) footage layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?; + let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| { + let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc); + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "shapesFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let snapshot = s.project.clone(); + let id = s.edit("Create Shapes from Text", None, |proj, st| { + let l = crate::vector::shapes_from_text(proj, &comp, &snapshot, cid, &src, time) + .ok_or_else(|| bad("layer.create", "the text layer has no glyphs"))?; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "masksFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let outlines = crate::vector::text_outlines_in_comp(&s.project, cid, &comp, &src, time); + if outlines.is_empty() { + skipped.push(format!("{}: no glyphs", src.name)); + continue; + } + let id = s.edit("Create Masks from Text", None, |proj, st| { + let name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = crate::vector::masks_layer(proj, &comp, &name, outlines, sid); + l.in_point = src.in_point; + l.out_point = src.out_point; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "editableText" => { + let Some((f, sl)) = footage.and_then(|f| f.layer.clone().map(|l| (f, l))) else { + skipped.push(format!("{}: not a Photoshop layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let id = s.edit("Convert to Editable Text", None, |proj, st| { + let l = crate::psd_import::editable_text(proj, &comp, &src, &psd, sl.index as usize) + .ok_or_else(|| bad("layer.create", format!("{}: the Photoshop layer has no text", src.name)))?; + let id = l.id; + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == lid).ok_or(EngineError::NoComp)?; + c.layers[at] = l; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + other => return Err(bad("layer.create", format!("`{other}` is not available yet (supported: {OPS})"))), + } + } + if created.is_empty() && !skipped.is_empty() { + return Err(bad("layer.create", skipped.join("; "))); + } + Ok(json!({"layers": created, "skipped": skipped})) +} + +/// Insert `l` directly above layer `below`; optionally switch the source's video off (as After +/// Effects does after converting). +fn insert_above( + proj: &mut effectcraft_project::Project, + cid: effectcraft_project::ItemId, + below: effectcraft_project::LayerId, + l: effectcraft_project::Layer, + hide_source: bool, +) -> Result<()> { + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == below).unwrap_or(0); + if hide_source && let Some(src) = c.layers.get_mut(at) { + src.switches.video = false; + } + c.layers.insert(at, l); + Ok(()) +} + +pub fn specs() -> Vec { + vec![cmd!("layer.create", "Create", [], None, "{op: editableText|shapesFromText|masksFromText|shapesFromVector, layers?}", has_layers, create)] +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd78..03cd80870 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,9 +525,41 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + let memory = matches!(what.as_str(), "all" | "memoryAndDisk" | "memory" | "image"); + let disk = matches!(what.as_str(), "all" | "memoryAndDisk" | "disk"); + if memory { + s.layer_cache.clear(); + } + let mut disk_entries = 0; + if disk && let Some(dc) = &s.disk_cache { + disk_entries = dc.stats().entries; + dc.clear(); + } + if matches!(what.as_str(), "all" | "image" | "memory" | "memoryAndDisk") { + effectcraft_effects::roto::purge(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); - Ok(json!({"purged": what})) + Ok(json!({"purged": what, "diskEntries": disk_entries})) +} + +/// `cache.diskStats`: the disk cache's folder, limit and contents. +pub(crate) fn disk_stats(s: &mut Session, _: &Value) -> Result { + let Some(dc) = &s.disk_cache else { return Ok(json!({"enabled": false})) }; + let st = dc.stats(); + Ok(json!({ + "enabled": true, + "folder": dc.folder().to_string_lossy(), + "entries": st.entries, + "frames": dc.keys(effectcraft_render::disk_cache::Kind::Frame).len(), + "layers": dc.keys(effectcraft_render::disk_cache::Kind::Layer).len(), + "bytes": st.bytes, + "maxBytes": st.max_bytes, + "hits": st.hits, + "misses": st.misses, + "writes": st.writes, + "evictions": st.evictions, + })) } /// Path of the footage behind the selected layer or project item. @@ -596,6 +628,7 @@ pub fn specs() -> Vec { cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group), cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge), cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), + crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), ] } diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 972e00eac..48285a33c 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,9 +84,47 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; + // Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes). + let import_as = str_p(p, "importAs").unwrap_or("footage"); + let retain = match import_as { + "footage" => None, + "composition" | "comp" => Some(false), + "compositionLayerSizes" | "compositionRetainLayerSizes" | "layerSizes" => Some(true), + other => return Err(bad("file.import", format!("importAs: footage|composition|compositionLayerSizes, not `{other}`"))), + }; + let mut paths = paths; + let mut out_comps = vec![]; let mut ids = vec![]; let mut errors = vec![]; + if let Some(retain) = retain { + let mut rest = vec![]; + for path in paths { + let bytes = match s.services.read_file(&path) { + Ok(b) if effectcraft_psd::is_psd(&b) => b, + _ => { + rest.push(path); + continue; + } + }; + match import_psd_comp(s, &path, bytes, retain) { + Ok((comp, items, warnings)) => { + out_comps.push(comp); + ids.extend(items); + errors.extend(warnings); + } + Err(e) => errors.push(format!("{path}: {e}")), + } + } + paths = rest; + if paths.is_empty() { + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + return Ok(json!({"items": ids, "comps": out_comps, "errors": errors})); + } + } let mut probed = vec![]; + let psd_layer = p.get("layer").cloned(); let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { // Data files (JSON, CSV, TSV) for data-driven animation: kept as text in the project. @@ -110,6 +148,18 @@ fn import(s: &mut Session, p: &Value) -> Result { f.frame_rate = r; f.duration = r.tick_of(frames.max(1)); } + // Choose Layer: one layer of a Photoshop document (document-sized). + if let Some(sel) = &psd_layer + && f.codec == "PSD" + { + match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) { + Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }), + None => { + errors.push(format!("{path}: no layer {sel}")); + continue; + } + } + } probed.push((path.clone(), f)) } Err(e) => errors.push(format!("{path}: {e}")), @@ -118,6 +168,10 @@ fn import(s: &mut Session, p: &Value) -> Result { s.edit("Import", None, |proj, st| { for (path, f) in probed { let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); + let name = match &f.layer { + Some(l) => format!("{}/{name}", l.name), + None => name, + }; let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, @@ -130,7 +184,37 @@ fn import(s: &mut Session, p: &Value) -> Result { } Ok(()) })?; - Ok(json!({"items": ids, "errors": errors})) + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + Ok(json!({"items": ids, "comps": out_comps, "errors": errors})) +} + +/// A Photoshop layer by index or name (pixel layers only). +fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, String)> { + let l = match sel { + Value::Number(n) => d.layers.get(n.as_u64()? as usize)?, + Value::String(name) => d.layers.iter().find(|l| &l.name == name)?, + _ => return None, + }; + Some((l.index, l.name.clone())) +} + +/// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings). +fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into()); + let rate = effectcraft_time::FrameRate::FPS_29_97; + let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 }; + let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs)); + let r = s.edit("Import", None, |proj, st| { + let r = crate::psd_import::import(proj, &psd, path, &name, retain, rate, duration); + st.project_selection = vec![r.comp]; + Ok(r) + })?; + let mut items = vec![r.folder.0]; + items.extend(r.items.iter().map(|i| i.0)); + Ok((r.comp.0, items, r.warnings)) } /// Extensions imported as data footage. @@ -255,7 +339,15 @@ pub fn specs() -> Vec { cmd!("file.saveAs", "Save As...", ["File", "Save As"], Some("Cmd+Shift+S"), "{path}", always, save_as), cmd!("file.incrementAndSave", "Increment and Save", ["File"], Some("Cmd+Alt+Shift+S"), "{}", has_path, increment_save), cmd!("file.revert", "Revert", ["File"], None, "{}", has_path, revert), - cmd!("file.import", "File...", ["File", "Import"], Some("Cmd+I"), "{paths: [string]}", always, import), + cmd!( + "file.import", + "File...", + ["File", "Import"], + Some("Cmd+I"), + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + always, + import + ), cmd!( "file.projectSettings", "Project Settings...", diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 45e6865ae..6908bcd56 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -359,9 +359,30 @@ fn collect_files(s: &mut Session, p: &Value) -> Result { // ---------------------------------------------------------------- scripts -/// Run a command script: a JSON array or JSON lines of `{"command": id, "params": {…}}` (also -/// accepts the control channel's `{"method": "engine.execute", "params": {"id", "params"}}`). +/// File ▸ Scripts ▸ Run Script File…: `.jsx` / `.js` files run as JavaScript against the After +/// Effects-style object model ([`Session::script`]); anything else is a command script: a JSON +/// array or JSON lines of `{"command": id, "params": {…}}` (also accepts the control channel's +/// `{"method": "engine.execute", "params": {"id", "params"}}`). fn run_script(s: &mut Session, p: &Value) -> Result { + if let Some(path) = str_p(p, "path") + && p.get("steps").is_none() + && is_js_script(path) + { + if path.to_ascii_lowercase().ends_with(".jsxbin") { + return Err(bad("file.runScript", "binary .jsxbin scripts are not supported: run the .jsx source")); + } + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let code = String::from_utf8_lossy(&bytes).to_string(); + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + let out = run_js(s, &code, path, false)?; + if let Some(e) = out.get("error").filter(|e| !e.is_null()) { + let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); + let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); + s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); + return Err(EngineError::Other(format!("{name}{line}: {msg}"))); + } + return Ok(out); + } let steps: Vec = match (p.get("steps"), str_p(p, "path")) { (Some(Value::Array(a)), _) => a.clone(), (_, Some(path)) => { @@ -389,7 +410,7 @@ fn run_script(s: &mut Session, p: &Value) -> Result { } _ => return Err(bad("file.runScript", format!("step {}: needs `command`", i + 1))), }; - if id == "file.runScript" { + if id == "file.runScript" || id == "script.run" { return Err(bad("file.runScript", "scripts can't run scripts")); } let r = s.execute(&id, params).map_err(|e| EngineError::Other(format!("step {} ({id}): {e}", i + 1)))?; @@ -398,6 +419,28 @@ fn run_script(s: &mut Session, p: &Value) -> Result { Ok(json!({"steps": results.len(), "results": results})) } +/// `.jsx` / `.js` (and `.jsxbin`, rejected) are JavaScript; `.json` / `.jsonl` / other files are +/// command scripts. +fn is_js_script(path: &str) -> bool { + let l = path.to_ascii_lowercase(); + l.ends_with(".jsx") || l.ends_with(".js") || l.ends_with(".jsxbin") +} + +/// Run JavaScript through the session's scripting engine. +fn run_js(s: &mut Session, code: &str, name: &str, console: bool) -> Result { + let runner = s.script.ok_or_else(|| EngineError::Other("JavaScript scripting is not available in this build".into()))?; + Ok(runner(s, &crate::ScriptRequest { code, name, console })) +} + +/// `script.run`: run JavaScript (the After Effects scripting object model: `app.project`, comps, +/// layers, properties…) and report `{ok, result, output, error}`; script errors are reported in +/// `error` (with line and column), not as a command failure. +fn script_run(s: &mut Session, p: &Value) -> Result { + let code = str_p(p, "code").ok_or_else(|| bad("script.run", "missing `code` (JavaScript)"))?.to_string(); + let name = str_p(p, "name").unwrap_or("script").to_string(); + run_js(s, &code, &name, super::b_p(p, "console").unwrap_or(false)) +} + // ---------------------------------------------------------------- footage /// Interpret Footage settings parsed from command parameters (every field optional). @@ -766,10 +809,19 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsonl/.json command script) | steps: [{command, params}]}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}]}", always, run_script ), + cmd!( + "script.run", + "Run Script", + [], + None, + "{code (JavaScript: app, app.project, CompItem, Layer, Property… like After Effects scripting), name?, console? (Script Console context)} → {ok, result, output, error: {message, line, column} | null}", + always, + script_run + ), cmd!( "file.interpretFootage", "Main...", diff --git a/crates/engine/src/commands/frontend.rs b/crates/engine/src/commands/frontend.rs index 39107f25e..f0068ae98 100644 --- a/crates/engine/src/commands/frontend.rs +++ b/crates/engine/src/commands/frontend.rs @@ -26,6 +26,9 @@ pub fn specs() -> Vec { ), fe!("app.gpuInfo", "GPU Information...", [], None, "{}", always), fe!("app.hide", "Hide EffectCraft", [], None, "{}", always), + // macOS: the native menu's Hide Others / Show All (the desktop app hides the other apps). + fe!("app.hideOthers", "Hide Others", [], None, "{}", always), + fe!("app.showAll", "Show All", [], None, "{}", always), fe!("app.quit", "Quit EffectCraft", [], Some("Cmd+Q"), "{}", always), fe!("app.commandPalette", "Quick Apply...", ["Edit"], Some("Cmd+Shift+Space"), "{query?}", always), fe!("app.keyboardShortcuts", "Keyboard Shortcuts", ["Edit"], Some("Cmd+Alt+'"), "{}", always), diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index 6f326b023..29b4c595d 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -436,6 +436,20 @@ fn rename(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// The Timeline's Comment column. +fn set_comment(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + let text = str_p(p, "comment").ok_or_else(|| bad("layer.setComment", "missing `comment`"))?.to_string(); + s.edit("Layer Comment", None, |proj, _| { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + for l in comp.layers.iter_mut().filter(|l| ids.contains(&l.id)) { + l.comment = text.clone(); + } + Ok(()) + })?; + Ok(Value::Null) +} + fn blend(s: &mut Session, p: &Value) -> Result { let (cid, ids) = layers_p(s, p)?; let mode = str_p(p, "mode").and_then(BlendMode::from_name); @@ -1077,6 +1091,7 @@ pub fn specs() -> Vec { set_switch ), cmd!("layer.rename", "Rename", [], Some("Enter"), "{layer?, name}", has_layers, rename), + cmd!("layer.setComment", "Layer Comment", [], None, "{layers?, comment}", has_layers, set_comment), cmd!("layer.setBlendMode", "Blending Mode", [], None, "{layers?, mode?: Normal|Multiply|Screen|…, step?: ±1}", has_layers, blend), cmd!( "layer.setTrackMatte", diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 7d44f5e52..56a585dd6 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,8 +4,10 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; +mod create; mod edit; mod effect; pub mod essential; @@ -29,10 +31,12 @@ pub mod paint; mod paths; mod project_items; mod prop; +mod prop_groups; mod proxy; pub mod puppet; mod query; mod render_queue; +pub mod roto_cmds; mod settings; mod stubs; mod styles; @@ -111,6 +115,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(text_edit::specs()); v.extend(layer_time::specs()); v.extend(prop::specs()); + v.extend(prop_groups::specs()); v.extend(project_items::specs()); v.extend(anim::specs()); v.extend(anim_tools::specs()); @@ -124,6 +129,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(help::specs()); v.extend(query::specs()); v.extend(layer_menu::specs()); + v.extend(create::specs()); v.extend(animation::specs()); v.extend(view::specs()); v.extend(viewer_cmds::specs()); @@ -135,6 +141,8 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); + v.extend(roto_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(liquify::specs()); diff --git a/crates/engine/src/commands/project_items.rs b/crates/engine/src/commands/project_items.rs index 832f5851a..a39498961 100644 --- a/crates/engine/src/commands/project_items.rs +++ b/crates/engine/src/commands/project_items.rs @@ -140,6 +140,92 @@ fn set_comment(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// Delete project items (folders with their contents). Layers that use a deleted item and Render +/// Queue items of deleted comps go too, like After Effects' Edit ▸ Clear in the Project panel. +fn delete(s: &mut Session, p: &Value) -> Result { + let c = "project.delete"; + let ids = items_p(s, p, c)?; + let mut doomed: Vec = vec![]; + for id in s.project.items.keys() { + if ids.iter().any(|d| is_within(&s.project, *id, *d)) { + doomed.push(*id); + } + } + let n = doomed.len(); + s.edit("Delete Items", None, |proj, st| { + for id in &doomed { + proj.items.remove(id); + } + let comp_ids: Vec = proj.comps().map(|(id, _)| *id).collect(); + for cid in comp_ids { + let uses = proj.comp(cid).is_some_and(|c| c.layers.iter().any(|l| l.source.item().is_some_and(|i| doomed.contains(&i)))); + if !uses { + continue; + } + if let Some(comp) = proj.comp_mut(cid) { + let gone: Vec = + comp.layers.iter().filter(|l| l.source.item().is_some_and(|i| doomed.contains(&i))).map(|l| l.id).collect(); + comp.layers.retain(|l| !gone.contains(&l.id)); + for l in &mut comp.layers { + if l.parent.is_some_and(|x| gone.contains(&x)) { + l.parent = None; + } + if l.track_matte.is_some_and(|m| gone.contains(&m.layer)) { + l.track_matte = None; + } + } + } + } + proj.render_queue.retain(|r| !doomed.contains(&r.comp)); + st.project_selection.retain(|i| !doomed.contains(i)); + Ok(()) + })?; + s.sanitize_state(); + Ok(json!(n)) +} + +/// Duplicate project items (Edit ▸ Duplicate in the Project panel): comps get fresh layer ids and +/// a numbered name; the copies are selected. +fn duplicate(s: &mut Session, p: &Value) -> Result { + let c = "project.duplicate"; + let ids = items_p(s, p, c)?; + let mut created = vec![]; + s.edit("Duplicate", None, |proj, st| { + for id in &ids { + let it = proj.item(*id).ok_or(EngineError::Other("item vanished".into()))?.clone(); + if it.is_folder() { + return Err(bad(c, "folders can't be duplicated")); + } + let base = it.name.trim_end_matches(|ch: char| ch.is_ascii_digit()).trim_end().to_string(); + let name = (2..).map(|i| format!("{base} {i}")).find(|n| proj.find_by_name(n).is_none()).unwrap_or_default(); + let mut kind = it.kind.clone(); + if let effectcraft_project::ItemKind::Comp(comp) = &mut kind { + let comp = std::sync::Arc::make_mut(comp); + let mut map = std::collections::HashMap::new(); + for l in &mut comp.layers { + let new = effectcraft_project::LayerId(proj.alloc()); + map.insert(l.id, new); + l.id = new; + let mut next = proj.next_id; + l.props.reassign_uids(&mut next); + proj.next_id = next + 1; + } + for l in &mut comp.layers { + l.parent = l.parent.and_then(|p| map.get(&p).copied()); + if let Some(m) = &mut l.track_matte { + m.layer = map.get(&m.layer).copied().unwrap_or(m.layer); + } + } + } + let nid = proj.add_item(&name, it.label, it.parent, kind); + created.push(nid); + } + st.project_selection = created.clone(); + Ok(()) + })?; + Ok(json!({"items": created.iter().map(|i| i.0).collect::>()})) +} + pub fn specs() -> Vec { vec![ cmd!("project.select", "Select Project Items", [], None, "{items: [id|name], add?}", always, select), @@ -147,5 +233,15 @@ pub fn specs() -> Vec { cmd!("project.move", "Move to Folder", [], None, "{items?: [id|name] (default: selected), folder: id|name|null (root)}", always, move_items), cmd!("project.setLabel", "Item Label", [], None, "{items?, label: name|index}", always, set_label), cmd!("project.setComment", "Item Comment", [], None, "{items?, comment}", always, set_comment), + cmd!( + "project.delete", + "Delete Project Items", + [], + None, + "{items?: [id|name] (default: selected)} (folders with their contents; layers using the items go too)", + always, + delete + ), + cmd!("project.duplicate", "Duplicate Project Items", [], None, "{items?: [id|name] (default: selected)} → {items}", always, duplicate), ] } diff --git a/crates/engine/src/commands/prop_groups.rs b/crates/engine/src/commands/prop_groups.rs new file mode 100644 index 000000000..7f5f2a2c2 --- /dev/null +++ b/crates/engine/src/commands/prop_groups.rs @@ -0,0 +1,107 @@ +//! Generic edits of property-tree groups by uid: rename, enable, remove, duplicate and reorder an +//! effect, mask, shape item or text animator (what scripting's `PropertyGroup.name`, `enabled`, +//! `remove()`, `duplicate()` and `moveTo()` do, and what agents need without a UI selection). + +use effectcraft_project::{GroupKind, ItemId, LayerId, Node, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, has_layers, layer_mut, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// Groups that can be renamed, removed, duplicated and reordered: instances in indexed lists. +fn is_instance(g: &PropGroup) -> bool { + matches!(g.kind, GroupKind::Indexed | GroupKind::Effect { .. } | GroupKind::Mask { .. } | GroupKind::Tracker { .. }) +} + +/// `{layer, prop: uid}` → (comp, layer, group uid), checking that the group is an instance. +fn group_ref(s: &Session, p: &Value, cmd: &str, need_instance: bool) -> Result<(ItemId, LayerId, Uid)> { + let (cid, lid) = layer_p(s, p, cmd)?; + let uid = p.get("prop").and_then(Value::as_u64).ok_or_else(|| bad(cmd, "missing `prop` (group uid)"))?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let g = l.props.find_group(uid).ok_or_else(|| bad(cmd, format!("no property group @{uid}")))?; + if need_instance && !is_instance(g) { + return Err(bad(cmd, format!("`{}` is a fixed group (only effects, masks, shape items and animators)", g.name))); + } + Ok((cid, lid, uid)) +} + +fn rename(s: &mut Session, p: &Value) -> Result { + let c = "prop.renameGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name`"))?.to_string(); + if name.trim().is_empty() { + return Err(bad(c, "the name can't be empty")); + } + s.edit("Rename", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + g.name = name.clone(); + Ok(json!(name)) + }) +} + +fn set_enabled(s: &mut Session, p: &Value) -> Result { + let c = "prop.setGroupEnabled"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let v = b_p(p, "value"); + s.edit("Toggle", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + g.enabled = v.unwrap_or(!g.enabled); + Ok(json!(g.enabled)) + }) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + let c = "prop.removeGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + s.edit("Delete", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + parent.children.retain(|n| n.uid() != uid); + st.selected_props.retain(|(_, u)| *u != uid); + Ok(Value::Null) + }) +} + +fn duplicate(s: &mut Session, p: &Value) -> Result { + let c = "prop.duplicateGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + s.edit("Duplicate", None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + let i = parent.children.iter().position(|n| n.uid() == uid).ok_or_else(|| bad(c, "group vanished"))?; + let Node::Group(mut g) = parent.children[i].clone() else { return Err(bad(c, "not a group")) }; + g.reassign_uids(&mut next); + let new = g.uid; + parent.children.insert(i + 1, Node::Group(g)); + proj.next_id = next + 1; + Ok(json!({"prop": new})) + }) +} + +fn move_to(s: &mut Session, p: &Value) -> Result { + let c = "prop.moveGroup"; + let (cid, lid, uid) = group_ref(s, p, c, true)?; + let to = p.get("index").and_then(Value::as_u64).ok_or_else(|| bad(c, "missing `index` (1-based)"))? as usize; + s.edit("Reorder", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let parent = l.props.parent_of_mut(uid).ok_or_else(|| bad(c, "group vanished"))?; + let i = parent.children.iter().position(|n| n.uid() == uid).ok_or_else(|| bad(c, "group vanished"))?; + if to < 1 || to > parent.children.len() { + return Err(bad(c, format!("index {to} is out of range 1..{}", parent.children.len()))); + } + let n = parent.children.remove(i); + parent.children.insert(to - 1, n); + Ok(json!(to)) + }) +} + +pub fn specs() -> Vec { + vec![ + cmd!("prop.renameGroup", "Rename Property Group", [], None, "{layer?, prop: uid, name}", has_layers, rename), + cmd!("prop.setGroupEnabled", "Enable Property Group", [], None, "{layer?, prop: uid, value?}", has_layers, set_enabled), + cmd!("prop.removeGroup", "Delete Property Group", [], None, "{layer?, prop: uid}", has_layers, remove), + cmd!("prop.duplicateGroup", "Duplicate Property Group", [], None, "{layer?, prop: uid} → {prop: new uid}", has_layers, duplicate), + cmd!("prop.moveGroup", "Reorder Property Group", [], None, "{layer?, prop: uid, index (1-based among its siblings)}", has_layers, move_to), + ] +} diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index acdf49127..91467ea52 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -135,7 +135,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -260,7 +260,7 @@ fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; it.output = OutputModule::for_format(f); } apply_settings(&mut it.settings, p, "renderQueue.add")?; @@ -438,7 +438,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { let n = it.extra_outputs.len() + 2; let mut om = it.output.clone(); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om = OutputModule::for_format(f); } if str_p(p, "output").is_none() && str_p(p, "path").is_none() { @@ -513,7 +513,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs new file mode 100644 index 000000000..95326b423 --- /dev/null +++ b/crates/engine/src/commands/roto_cmds.rs @@ -0,0 +1,424 @@ +//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer +//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. +//! +//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the +//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base +//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze +//! first). Every edit is one undo step. + +use effectcraft_effects::roto::{self as fx, FROZEN, Stroke, StrokeKind}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::Ids; +use effectcraft_project::{ItemId, Layer, LayerId, PropGroup, Uid}; +use effectcraft_track::roto::{RotoData, iou, rle}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::roto::{self as rt, Direction, RotoOptions, RotoTask, VIEWS, is_roto, params_static}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "roto.stroke", + "Roto Brush Stroke", + [], + None, + "{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?}", + has_comp, + stroke + ), + cmd!( + "roto.propagate", + "Propagate Roto Brush", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, to? (layer frame: extend the span), wait?: block until done}", + has_comp, + propagate + ), + cmd!("roto.span", "Set Segmentation Span", [], None, "{layer?, effect?, start?, end? (layer frames)}", has_comp, span), + cmd!("roto.freeze", "Freeze", [], None, "{layer?, effect?, wait?}", has_comp, freeze), + cmd!("roto.unfreeze", "Unfreeze", [], None, "{layer?, effect?}", has_comp, unfreeze), + cmd!("roto.clearStrokes", "Remove Roto Brush Strokes", [], None, "{layer?, effect?, frame? (only this layer frame), kind?}", has_comp, clear_strokes), + cmd!("roto.cancel", "Stop Roto Brush", [], None, "{}", is_running, |s, _| Ok(json!({"stopped": s.stop_roto()}))), + cmd!( + "roto.options", + "Roto Brush Options", + [], + None, + "{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?}", + always, + options + ), + query!( + "roto.status", + "Roto Brush Status", + "{layer?, effect?, frame? (layer frame: also report its matte), matte?: include the matte (RLE, base64), compute?: compute the frame if needed, compareTo?: RLE matte to report the IoU against}", + status + ), + ] +} + +fn is_running(s: &Session) -> std::result::Result<(), String> { + if s.is_roto_running() { Ok(()) } else { Err("no Roto Brush job is running".into()) } +} + +/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), +/// else a selected one, else the first on the selected (or any) layer. +pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fxg) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fxg.groups().find(|g| g.uid == n).or_else(|| fxg.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fxg.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is_roto(g))), + }; + if let Some(g) = g.filter(|g| is_roto(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let g = l.props.find_group(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + Ok((l, g)) +} + +fn data_of(g: &PropGroup) -> RotoData { + fx::data(¶ms_static(g)).as_ref().clone() +} + +fn frozen_of(g: &PropGroup) -> bool { + matches!(g.get(FROZEN).map(|p| &p.value), Some(KV::Str(s)) if !s.is_empty()) +} + +/// Write an instance's data (one undo step). +fn write_data(s: &mut Session, label: &str, cid: ItemId, lid: LayerId, uid: Uid, d: &RotoData) -> Result<()> { + let j = d.to_json(); + s.edit(label, None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(j); + } + Ok(()) + }) +} + +/// Current layer frame of `layer` (or `frame` from the params). +fn frame_p(s: &Session, p: &Value, cid: ItemId, layer: &Layer) -> i64 { + if let Some(f) = f_p(p, "frame") { + return f.round() as i64; + } + let comp = s.project.comp(cid); + let fps = comp.map(|c| c.frame_rate.as_f64()).unwrap_or(30.0); + fx::frame_of(layer.layer_time(s.time_of(cid)).seconds(), fps) +} + +fn points_p(p: &Value, cmd: &str) -> Result> { + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer pixels)"))?; + let mut v = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + v.push([x, y]); + } + if v.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + Ok(v) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.stroke"; + let (cid, lid) = match roto_p(s, p, cmd) { + Ok((c, l, _)) => (c, l), + Err(_) => layer_p(s, p, cmd)?, + }; + let kind = match str_p(p, "kind") { + Some(k) => StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (fg | bg | refine | refineErase)")))?, + None => StrokeKind::Fg, + }; + let points = points_p(p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "the Roto Brush works on footage, solid and composition layers")); + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + let o = &s.state.roto; + let def_r = if matches!(kind, StrokeKind::Refine | StrokeKind::RefineErase) { o.refine_diameter / 2.0 } else { o.diameter / 2.0 }; + let radius = f_p(p, "radius").unwrap_or(def_r).clamp(0.5, 2500.0); + let size = rt::layer_size(&s.project, comp, layer); + // The instance: the requested / first one, else a new one. + let existing = roto_p(s, p, cmd).ok().filter(|(c, l, _)| *c == cid && *l == lid).map(|x| x.2); + if let Some(uid) = existing { + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + } + if existing.is_none() && !kind.is_segmentation() { + return Err(bad(cmd, "draw a foreground stroke with the Roto Brush tool first")); + } + let st = Stroke { kind, frame, radius, points }; + let label = if kind.is_segmentation() { "Roto Brush Stroke" } else { "Refine Edge Stroke" }; + let (uid, d) = s.edit(label, None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let uid = match existing { + Some(u) => u, + None => { + let spec = effectcraft_effects::find(fx::ID).ok_or_else(|| EngineError::Other("Roto Brush effect missing".into()))?; + let n = fxg.groups().filter(|g| is_roto(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let u = g.uid; + fxg.children.push(g.into()); + u + } + }; + let g = fxg.find_group_mut(uid).ok_or_else(|| bad(cmd, "effect gone"))?; + let mut d = data_of(g); + d.add(st, fx::DEFAULT_SPAN, limits); + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(d.to_json()); + } + proj.next_id = proj.next_id.max(next); + Ok((uid, d)) + })?; + if !s.state.selected_props.iter().any(|(l, u)| *l == lid && *u == uid) { + s.state.selected_layers = vec![lid]; + } + Ok(json!({ + "layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius, + "base": d.base, "span": d.span, "strokes": d.strokes.len(), + })) +} + +fn propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.propagate"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + if let Some(to) = f_p(p, "to") { + let to = to.round() as i64; + let (l, g) = group(s, cid, lid, uid)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let to = to.clamp(lim[0], lim[1]); + let before = d.span; + d.span[0] = d.span[0].min(to); + d.span[1] = d.span[1].max(to); + if d.span != before { + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + write_data(s, "Extend Segmentation Span", cid, lid, uid, &d)?; + } + } + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn span(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.span"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (l, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let base = d.base.ok_or_else(|| bad(cmd, "no strokes yet"))?; + let first = d.strokes.iter().map(|s| s.frame).min().unwrap_or(base); + let last = d.strokes.iter().map(|s| s.frame).max().unwrap_or(base); + if let Some(a) = f_p(p, "start") { + d.span[0] = (a.round() as i64).clamp(lim[0], first); + } + if let Some(b) = f_p(p, "end") { + d.span[1] = (b.round() as i64).clamp(last, lim[1]); + } + write_data(s, "Segmentation Span", cid, lid, uid, &d)?; + Ok(json!({"layer": lid.0, "effect": uid, "span": d.span, "base": base})) +} + +fn freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.freeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.unfreeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if !frozen_of(g) { + return Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})); + } + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn clear_strokes(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.clearStrokes"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + let mut d = data_of(g); + let frame = f_p(p, "frame").map(|f| f.round() as i64); + let kind = match str_p(p, "kind") { + Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?), + None => None, + }; + let n0 = d.strokes.len(); + d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k))); + if d.strokes.iter().all(|s| !s.kind.is_segmentation()) { + d.base = None; + } else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) { + // The base moves to the earliest remaining stroked frame. + d.base = d.strokes.iter().filter(|s| s.kind.is_segmentation()).map(|s| s.frame).min(); + } + let removed = n0 - d.strokes.len(); + if removed > 0 { + write_data(s, "Remove Roto Brush Strokes", cid, lid, uid, &d)?; + } + Ok(json!({"layer": lid.0, "effect": uid, "removed": removed, "strokes": d.strokes.len(), "base": d.base})) +} + +fn options(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.options"; + let mut o: RotoOptions = s.state.roto.clone(); + for (k, f) in [("diameter", &mut o.diameter as &mut f64), ("refineDiameter", &mut o.refine_diameter), ("overlayOpacity", &mut o.overlay_opacity)] { + if let Some(v) = f_p(p, k) { + *f = v; + } + } + o.diameter = o.diameter.clamp(1.0, 5000.0); + o.refine_diameter = o.refine_diameter.clamp(1.0, 5000.0); + o.overlay_opacity = o.overlay_opacity.clamp(0.0, 100.0); + if let Some(v) = str_p(p, "view") { + let v = VIEWS.iter().find(|x| x.eq_ignore_ascii_case(v)).ok_or_else(|| bad(cmd, format!("unknown view `{v}` ({})", VIEWS.join(" | "))))?; + o.view = v.to_string(); + } + for (k, c) in [("overlayColor", &mut o.overlay_color), ("boundaryColor", &mut o.boundary_color)] { + if let Some(a) = p.get(k).and_then(Value::as_array) { + if a.len() < 3 { + return Err(bad(cmd, format!("`{k}` is [r, g, b] (0–1)"))); + } + for (i, v) in a.iter().take(3).enumerate() { + c[i] = v.as_f64().unwrap_or(0.0).clamp(0.0, 1.0); + } + } + } + if let Some(b) = b_p(p, "autoPropagate") { + o.auto_propagate = b; + } + s.state.roto = o.clone(); + Ok(serde_json::to_value(o).unwrap_or(Value::Null)) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let mut out = + json!({"running": s.is_roto_running(), "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner()), "options": s.state.roto}); + let Ok((cid, lid, uid)) = roto_p(s, p, "roto.status") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let (layer, g) = group(s, cid, lid, uid)?; + let (layer, g) = (layer.clone(), g.clone()); + let d = data_of(&g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["base"] = json!(d.base); + out["span"] = json!(d.span); + out["strokes"] = json!(d.strokes.len()); + out["strokeFrames"] = json!(d.stroke_frames()); + out["frozen"] = json!(frozen_of(&g)); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["limits"] = json!(rt::frame_limits(&comp, &layer)); + let frame = frame_p(s, p, cid, &layer); + out["frame"] = json!(frame); + let want_matte = b_p(p, "matte").unwrap_or(false); + let compute = b_p(p, "compute").unwrap_or(want_matte); + if (p.get("frame").is_some() || want_matte) && d.in_span(frame) { + let size = rt::layer_size(&s.project, &comp, &layer); + let fps = comp.frame_rate.as_f64(); + let ps = params_static(&g); + let chain = fx::Chain::new(&ps, size, fps, 1.0); + let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) { + Some(sg) => Some(sg), + None if compute => { + let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0); + let mut src = rt::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, cid, &comp, &layer, index, size); + chain.ensure(frame, &mut |f| src.get(f), &|| false) + } + None => None, + }; + if let Some(sg) = seg { + out["area"] = json!(sg.area()); + out["centroid"] = json!(sg.centroid()); + out["size"] = json!([sg.w, sg.h]); + out["refineArea"] = json!(sg.refine.iter().filter(|v| **v != 0).count()); + if want_matte { + out["matte"] = json!(rle::encode(&sg.matte)); + } + if let Some(gt) = p.get("compareTo").and_then(Value::as_str).and_then(rle::decode) { + out["iou"] = json!(iou(&sg.matte, >)); + } + } + } + let computed = s.roto_computed(cid, lid, uid); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + Ok(out) +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index b62f36546..764d6d99f 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -17,16 +17,12 @@ macro_rules! stub { pub fn specs() -> Vec { vec![ - // macOS application menu items handled by the OS. - stub!("app.hideOthers", "Hide Others", [], "{}"), - stub!("app.showAll", "Show All", [], "{}"), stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), - stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). @@ -48,7 +44,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/commands/text_anim.rs b/crates/engine/src/commands/text_anim.rs index 73f34c5e9..51367da91 100644 --- a/crates/engine/src/commands/text_anim.rs +++ b/crates/engine/src/commands/text_anim.rs @@ -114,7 +114,8 @@ fn add_animator(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("Animator {}", anims.children.len() + 1)); let mut g = build::text_animator(&mut ids, &name, vec![]); let added: usize = kinds.iter().map(|k| add_props(&mut ids, &mut g, k, three)).sum(); - if added == 0 { + // `properties: []` adds an empty animator (scripting's addProperty("ADBE Text Animator")). + if added == 0 && !kinds.is_empty() { return Err(bad(C, "unknown animator property")); } let uid = g.uid; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b2c4d342e..2d5beb792 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -16,10 +17,13 @@ pub mod links; pub mod mask_track; pub mod menus; pub mod prefs; +pub mod psd_import; pub mod render_queue; +pub mod roto; mod session_settings; pub mod shortcuts; pub mod tracking; +pub mod vector; pub mod viewer; pub mod warp; @@ -41,6 +45,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,12 +175,18 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, /// Paint and Brushes panel options (Brush, Clone Stamp and Eraser tools). #[serde(default)] pub paint: commands::paint::PaintOptions, + /// Roto Brush and Refine Edge tool options. + #[serde(default)] + pub roto: roto::RotoOptions, /// Puppet tool options for new meshes. #[serde(default)] pub puppet: commands::puppet::PuppetOptions, @@ -270,11 +281,21 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) Roto Brush propagation / Freeze. + pub roto_job: Option, + /// Roto Brush instances edited since their last propagation: (comp, layer, effect uid). + pub roto_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, /// Processed-layer pixels reused across frames and edits (content-keyed, never stale). pub layer_cache: Arc, + /// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled. + pub disk_cache: Option>, /// Settings (Preferences). pub prefs: prefs::Prefs, /// Bumped whenever settings change (frontends re-apply theme, labels…). @@ -292,8 +313,25 @@ pub struct Session { pub autosave: autosave::AutoSaveState, /// The viewer snapshot (Take Snapshot / Show Snapshot). pub snapshot: Option, + /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): + /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. + pub script: Option, } +/// A script to run (see [`Session::script`]). +#[derive(Clone, Copy, Debug)] +pub struct ScriptRequest<'a> { + pub code: &'a str, + /// File name shown in errors and `$.fileName` (`console` for the Script Console). + pub name: &'a str, + /// Run in the Script Console's persistent context (variables survive between runs). + pub console: bool, +} + +/// Runs a script against the session and reports `{ok, result, output, error: {message, line, +/// column, file} | null}`. +pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; + impl Default for Session { fn default() -> Self { Session { @@ -315,9 +353,14 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], + roto_job: None, + roto_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), + disk_cache: None, prefs: prefs::Prefs::default(), prefs_revision: 0, config: None, @@ -326,6 +369,7 @@ impl Default for Session { shortcut_table: std::sync::OnceLock::new(), autosave: autosave::AutoSaveState::default(), snapshot: None, + script: None, } } } @@ -384,6 +428,18 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } + // Roto Brush Input Keys follow the frames; edited instances propagate again. + for r in roto::sync(&before, &mut p) { + if !self.roto_pending.contains(&r) { + self.roto_pending.push(r); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -403,6 +459,9 @@ impl Session { pub fn bump(&mut self) { self.revision += 1; + if self.disk_cache.is_some() { + self.layer_cache.set_disk_salt(effectcraft_render::disk_cache::footage_salt(&self.project)); + } self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -561,6 +620,12 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); + self.stop_roto(); + self.roto_job = None; + self.roto_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -584,6 +649,10 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] +mod tests_disk_cache; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_essential; @@ -618,6 +687,8 @@ mod tests_timeline; #[cfg(test)] mod tests_track; #[cfg(test)] +mod tests_vector_import; +#[cfg(test)] mod tests_viewer; /// Font families available to text layers (bundled + scanned system fonts). @@ -631,3 +702,5 @@ pub fn text_families() -> Vec { } #[cfg(test)] mod tests_paint; +#[cfg(test)] +mod tests_roto; diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 6177e637d..66d198948 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -17,8 +17,9 @@ //! //! An entry's shortcut is the command's default shortcut unless the line gives one (needed for //! entries with bound parameters). Adobe-service entries (Team Projects, Libraries, Bridge, Media -//! Encoder, Behance, Creative Cloud…) are intentionally absent; Essential Graphics templates use -//! EffectCraft's own open `.ectemplate` format. +//! Encoder, Behance, Creative Cloud…) are intentionally absent; the Essential Graphics workspace +//! uses the Properties panel and the Essential Graphics panel, whose templates use EffectCraft's +//! own open `.ectemplate` format. use std::sync::OnceLock; @@ -913,16 +914,22 @@ View Enter Full Screen | view.fullScreen Window Workspace + Default | window.workspace {"name":"Default"} | Shift+F10 + Review | window.workspace {"name":"Review"} + Learn | window.workspace {"name":"Learn"} + Small Screen | window.workspace {"name":"Small Screen"} | Shift+F12 + Standard | window.workspace {"name":"Standard"} | Shift+F11 + --- All Panels | window.workspace {"name":"All Panels"} Animation | window.workspace {"name":"Animation"} - Default | window.workspace {"name":"Default"} | Shift+F10 + Color | window.workspace {"name":"Color"} Effects | window.workspace {"name":"Effects"} + Essential Graphics | window.workspace {"name":"Essential Graphics"} Minimal | window.workspace {"name":"Minimal"} Motion Tracking | window.workspace {"name":"Motion Tracking"} Paint | window.workspace {"name":"Paint"} - Small Screen | window.workspace {"name":"Small Screen"} | Shift+F12 - Standard | window.workspace {"name":"Standard"} | Shift+F11 Text | window.workspace {"name":"Text"} + Undocked Panels | window.workspace {"name":"Undocked Panels"} --- Reset to Saved Layout | window.resetWorkspace Save Changes to this Workspace | window.saveWorkspace @@ -947,6 +954,7 @@ Window Preview | window.panel {"panel":"preview"} | Cmd+3 Progress | window.unavailablePanel {"panel":"progress"} Properties | window.panel {"panel":"properties"} + Script Console | window.panel {"panel":"scriptConsole"} Smoother | window.panel {"panel":"smoother"} Tools | window.panel {"panel":"tools"} | Cmd+1 Tracker | window.panel {"panel":"tracker"} diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index ec06b3f0b..5a2afab8d 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -115,6 +115,8 @@ page!(Appearance { use_label_color_for_tabs: bool = true, cycle_mask_colors: bool = true, use_gradients: bool = true, + /// macOS: draw the menu bar inside the window instead of the system menu bar. + in_window_menu_bar_mac: bool = false, }); page!(Grids { @@ -740,6 +742,8 @@ pub fn pages() -> Vec { s("appearance.useLabelColorForTabs", "Use Label Color for Related Tabs", B, false), s("appearance.cycleMaskColors", "Cycle Mask Colors", B, false), s("appearance.useGradients", "Use Gradients", B, false), + Section("Menu Bar"), + s("appearance.inWindowMenuBarMac", "Use In-Window Menu Bar on macOS", B, true), ], }, Page { diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs new file mode 100644 index 000000000..7a0a79626 --- /dev/null +++ b/crates/engine/src/psd_import.rs @@ -0,0 +1,635 @@ +//! Photoshop import as a composition (File ▸ Import ▸ File… with Import As: Composition or +//! Composition – Retain Layer Sizes), and Layer ▸ Create ▸ Convert to Editable Text. +//! +//! Every pixel layer becomes a footage item that names its layer (`Footage::layer`), placed in +//! a composition the size of the document. Groups become precomps (pass-through groups collapse +//! so their blend modes act on the layers below, like Photoshop); type layers become editable +//! text layers; layer masks are baked into the layer's alpha and vector masks become masks; +//! layer effects map to Layer Styles; adjustment layers become adjustment layers with the +//! matching effect (best effort); clipped layers preserve the transparency underneath. + +use std::sync::Arc; + +use effectcraft_color::BlendMode; +use effectcraft_color::Label; +use effectcraft_keyframe::{Gradient, Justify, ShapePath, TextDoc, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::styles::{self, STYLE_BLEND_MODES}; +use effectcraft_project::{AlphaMode, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, Project, PropGroup, Solid, SourceLayer}; +use effectcraft_psd::{Adjustment, DValue, Descriptor, Node, Psd}; +use effectcraft_time::{FrameRate, Tick}; + +/// Photoshop blend mode key → compositor mode (`pass` = pass through, handled by collapsing). +pub fn blend_mode(key: &str) -> BlendMode { + match key { + "diss" => BlendMode::Dissolve, + "dark" => BlendMode::Darken, + "mul " => BlendMode::Multiply, + "idiv" => BlendMode::ColorBurn, + "lbrn" => BlendMode::LinearBurn, + "dkCl" => BlendMode::DarkerColor, + "lite" => BlendMode::Lighten, + "scrn" => BlendMode::Screen, + "div " => BlendMode::ColorDodge, + "lddg" => BlendMode::LinearDodge, + "lgCl" => BlendMode::LighterColor, + "over" => BlendMode::Overlay, + "sLit" => BlendMode::SoftLight, + "hLit" => BlendMode::HardLight, + "vLit" => BlendMode::VividLight, + "lLit" => BlendMode::LinearLight, + "pLit" => BlendMode::PinLight, + "hMix" => BlendMode::HardMix, + "diff" => BlendMode::Difference, + "smud" => BlendMode::Exclusion, + "fsub" => BlendMode::Subtract, + "fdiv" => BlendMode::Divide, + "hue " => BlendMode::Hue, + "sat " => BlendMode::Saturation, + "colr" => BlendMode::Color, + "lum " => BlendMode::Luminosity, + _ => BlendMode::Normal, + } +} + +/// Descriptor blend mode enum (`BlnM`) → compositor mode. +fn desc_blend(v: Option<&str>) -> Option { + Some(match v? { + "Nrml" => BlendMode::Normal, + "Dslv" => BlendMode::Dissolve, + "Drkn" => BlendMode::Darken, + "Mltp" => BlendMode::Multiply, + "CBrn" => BlendMode::ColorBurn, + "linearBurn" => BlendMode::LinearBurn, + "darkerColor" => BlendMode::DarkerColor, + "Lghn" => BlendMode::Lighten, + "Scrn" => BlendMode::Screen, + "CDdg" => BlendMode::ColorDodge, + "linearDodge" => BlendMode::LinearDodge, + "lighterColor" => BlendMode::LighterColor, + "Ovrl" => BlendMode::Overlay, + "SftL" => BlendMode::SoftLight, + "HrdL" => BlendMode::HardLight, + "vividLight" => BlendMode::VividLight, + "linearLight" => BlendMode::LinearLight, + "pinLight" => BlendMode::PinLight, + "hardMix" => BlendMode::HardMix, + "Dfrn" => BlendMode::Difference, + "Xclu" => BlendMode::Exclusion, + "blendSubtraction" => BlendMode::Subtract, + "blendDivide" => BlendMode::Divide, + "H " => BlendMode::Hue, + "Strt" => BlendMode::Saturation, + "Clr " => BlendMode::Color, + "Lmns" => BlendMode::Luminosity, + _ => return None, + }) +} + +/// What the import created. +pub struct PsdImport { + pub comp: ItemId, + pub folder: ItemId, + pub items: Vec, + pub warnings: Vec, +} + +struct Ctx<'a> { + psd: &'a Psd, + path: &'a str, + retain: bool, + folder: ItemId, + rate: FrameRate, + duration: Tick, + items: Vec, + warnings: Vec, +} + +/// Import `psd` (read from `path`) as a composition named `name`. +pub fn import(proj: &mut Project, psd: &Psd, path: &str, name: &str, retain: bool, rate: FrameRate, duration: Tick) -> PsdImport { + let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder); + let mut cx = Ctx { psd, path, retain, folder, rate, duration, items: vec![], warnings: vec![] }; + let comp = build_comp(proj, &mut cx, name, &psd.tree(), None); + PsdImport { comp, folder, items: cx.items, warnings: cx.warnings } +} + +fn new_comp(cx: &Ctx) -> Comp { + Comp::new(cx.psd.width, cx.psd.height, cx.rate, cx.duration) +} + +fn build_comp(proj: &mut Project, cx: &mut Ctx, name: &str, nodes: &[Node], parent: Option) -> ItemId { + let cid = proj.add_item(name, Label::Sandstone, parent, ItemKind::Comp(Arc::new(new_comp(cx)))); + cx.items.push(cid); + let mut comp = new_comp(cx); + for n in nodes { + if let Some(l) = build_layer(proj, cx, &comp, n) { + comp.layers.push(l); + } + } + styles::sync_comp(&mut comp); + if let Some(item) = proj.item_mut(cid) { + item.kind = ItemKind::Comp(Arc::new(comp)); + } + cid +} + +fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> (ItemId, (u32, u32)) { + let l = &cx.psd.layers[index]; + let retain = cx.retain; + let (w, h) = if retain && !l.rect.is_empty() { (l.rect.width(), l.rect.height()) } else { (cx.psd.width, cx.psd.height) }; + let file = std::path::Path::new(cx.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let f = Footage { + path: cx.path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: cx.rate, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PSD".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }), + ..Default::default() + }; + let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); + cx.items.push(id); + (id, (w, h)) +} + +fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option { + let (index, children) = match n { + Node::Layer(i) => (*i, None), + Node::Group { layer, children } => (*layer, Some(children)), + }; + let pl = cx.psd.layers[index].clone(); + let name = if pl.name.is_empty() { format!("Layer {}", index + 1) } else { pl.name.clone() }; + let mut layer = if let Some(children) = children { + let sub = build_comp(proj, cx, &name, children, Some(cx.folder)); + let mut l = build::layer(proj, comp, &name, LayerSource::Comp { item: sub }, (comp.width, comp.height), None); + // Pass-through groups composite their layers straight into the parent. + l.switches.collapse = pl.blend_key() == "pass"; + l + } else if let Some(t) = &pl.text { + text_layer(proj, comp, &name, t) + } else if let Some(adj) = &pl.adjustment { + let mut l = adjustment_layer(proj, comp, &name); + add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name); + l + } else if pl.has_pixels() || pl.fill.is_some() { + let (item, size) = footage_item(proj, cx, index, &name); + let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None); + if cx.retain && !pl.rect.is_empty() { + set_vec2(&mut l, "transform/position", [pl.rect.left as f64 + size.0 as f64 / 2.0, pl.rect.top as f64 + size.1 as f64 / 2.0]); + } + l + } else { + // Empty layers have nothing to show. + return None; + }; + layer.name = comp.unique_layer_name(&name); + layer.blend_mode = blend_mode(pl.blend_key()); + layer.switches.video = !pl.hidden; + layer.preserve_transparency = pl.clipping; + set_scalar(&mut layer, "transform/opacity", pl.opacity as f64 / 255.0 * 100.0); + // Vector mask → a mask in layer space. + let origin = layer_origin(&pl, cx.retain, &layer); + if let Some(vm) = pl.vector_mask.as_ref().filter(|v| !v.disabled) + && let Some(masks) = layer.props.sub_mut("masks") + { + let mut ids = Ids(&mut proj.next_id); + for (k, sp) in vm.subpaths.iter().enumerate() { + let n = sp.knots.len(); + let mut path = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: sp.closed }; + for kn in &sp.knots { + let v = [kn[1][0] - origin[0], kn[1][1] - origin[1]]; + path.vertices.push(v); + path.in_tangents.push([kn[0][0] - kn[1][0], kn[0][1] - kn[1][1]]); + path.out_tangents.push([kn[2][0] - kn[1][0], kn[2][1] - kn[1][1]]); + } + if n == 0 { + continue; + } + let mode = if vm.inverted { MaskMode::Subtract } else { MaskMode::Add }; + masks.children.push(build::mask(&mut ids, &format!("Mask {}", k + 1), path, mode, [255, 255, 0]).into()); + } + } + // Layer effects → Layer Styles; Fill Opacity → Advanced Blending. + let gl = comp.global_light.clone(); + if let Some(fx) = &pl.effects + && fx.bool("masterFXSwitch") != Some(false) + && layer.can_have_styles() + { + apply_effects(proj, &mut layer, fx, &gl); + } + if pl.fill_opacity < 255 && layer.can_have_styles() { + let mut ids = Ids(&mut proj.next_id); + if layer.props.sub(styles::GROUP).is_none() { + let g = styles::layer_styles(&mut ids, &gl); + let at = layer.props.children.iter().position(|c| c.match_id() == "transform").map(|i| i + 1).unwrap_or(layer.props.children.len()); + layer.props.children.insert(at, g.into()); + } + if let Some(p) = layer.props.prop_mut("layerStyles/blendingOptions/advancedBlending/fillOpacity") { + p.value = Value::Scalar(pl.fill_opacity as f64 / 255.0 * 100.0); + } + } + Some(layer) +} + +/// Where layer space's origin sits in document pixels. +fn layer_origin(pl: &effectcraft_psd::Layer, retain: bool, layer: &Layer) -> [f64; 2] { + if let Some(t) = &pl.text { + return [t.transform[4], t.transform[5]]; + } + if matches!(layer.source, LayerSource::Footage { .. }) && retain && !pl.rect.is_empty() { + return [pl.rect.left as f64, pl.rect.top as f64]; + } + [0.0, 0.0] +} + +fn set_scalar(l: &mut Layer, path: &str, v: f64) { + if let Some(p) = l.props.prop_mut(path) { + p.value = Value::Scalar(v); + } +} + +fn set_vec2(l: &mut Layer, path: &str, v: [f64; 2]) { + if let Some(p) = l.props.prop_mut(path) { + p.value = match p.value { + Value::Vec3(o) => Value::Vec3([v[0], v[1], o[2]]), + _ => Value::Vec2(v), + }; + } +} + +/// Split a PostScript font name into family and style (`Inter-Bold` → `Inter`, `Bold`). +fn font_names(ps: &str) -> (String, String) { + match ps.split_once('-') { + Some((fam, style)) => (fam.to_string(), style.to_string()), + None => (ps.to_string(), "Regular".to_string()), + } +} + +fn text_doc(t: &effectcraft_psd::TextInfo) -> TextDoc { + let mut d = TextDoc::default(); + d.set_text(&t.text); + let [xx, xy, yx, yy, _, _] = t.transform; + let scale = (xx * yy - xy * yx).abs().sqrt().max(1e-6); + if let Some(s) = t.style.size { + d.size = s * scale; + } + if let Some(f) = &t.style.font { + let (fam, style) = font_names(f); + d.font = fam; + d.style = style; + } + if let Some(c) = t.style.color { + d.fill = [c[0] as f32, c[1] as f32, c[2] as f32, 1.0]; + } + d.justify = match t.style.justification { + Some(1) => Justify::Right, + Some(2) => Justify::Center, + Some(3) => Justify::JustifyLastLeft, + Some(4) => Justify::JustifyLastRight, + Some(5) => Justify::JustifyLastCenter, + Some(6) => Justify::JustifyAll, + _ => Justify::Left, + }; + if let Some(tr) = t.style.tracking { + d.tracking = tr; + } + if let Some(l) = t.style.leading { + d.leading = Some(l * scale); + } + d.faux_bold = t.style.faux_bold; + d.faux_italic = t.style.faux_italic; + if t.style.box_text + && let Some([l, top, r, b]) = t.bounds + { + d.box_size = Some([(r - l) * scale, (b - top) * scale]); + d.box_pos = [l * scale, top * scale]; + } + d +} + +fn text_layer(proj: &mut Project, comp: &Comp, name: &str, t: &effectcraft_psd::TextInfo) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Text, (0, 0), None); + if let Some(p) = l.props.prop_mut("text/sourceText") { + p.value = Value::Text(Box::new(text_doc(t))); + } + set_vec2(&mut l, "transform/position", [t.transform[4], t.transform[5]]); + let [xx, xy, _, _, _, _] = t.transform; + let rot = xy.atan2(xx).to_degrees(); + if rot.abs() > 1e-6 { + set_scalar(&mut l, "transform/rotation", rot); + } + l +} + +fn adjustment_layer(proj: &mut Project, comp: &Comp, name: &str) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (comp.width, comp.height), None); + l.switches.adjustment = true; + l.label = Label::Purple; + l +} + +fn add_effect(proj: &mut Project, l: &mut Layer, effect: &str, comp: &Comp, params: &[(&str, Value)]) { + let Some(spec) = effectcraft_effects::lookup(effect) else { return }; + let mut ids = Ids(&mut proj.next_id); + let mut g = effectcraft_effects::instantiate(spec, &mut ids, spec.name, [comp.width as f64, comp.height as f64]); + for (k, v) in params { + if let Some(p) = g.get_mut(k) { + p.value = v.clone(); + } + } + if let Some(fx) = l.props.sub_mut("effects") { + fx.children.push(g.into()); + } +} + +fn add_adjustment_effect(proj: &mut Project, l: &mut Layer, adj: &Adjustment, comp: &Comp, warnings: &mut Vec, name: &str) { + match adj { + Adjustment::Invert => add_effect(proj, l, "Invert", comp, &[]), + Adjustment::BrightnessContrast { brightness, contrast, .. } => { + add_effect(proj, l, "Brightness & Contrast", comp, &[("brightness", Value::Scalar(*brightness)), ("contrast", Value::Scalar(*contrast))]) + } + Adjustment::HueSaturation { hue, saturation, lightness, colorize } => { + let mut p = vec![("hue", Value::Scalar(*hue)), ("saturation", Value::Scalar(*saturation)), ("lightness", Value::Scalar(*lightness))]; + if *colorize { + p = vec![ + ("colorize", Value::Bool(true)), + ("colorizeHue", Value::Scalar(*hue)), + ("colorizeSaturation", Value::Scalar(*saturation)), + ("colorizeLightness", Value::Scalar(*lightness)), + ]; + } + add_effect(proj, l, "Hue/Saturation", comp, &p) + } + Adjustment::Levels { in_black, in_white, gamma, out_black, out_white } => add_effect( + proj, + l, + "Levels", + comp, + &[ + ("inBlack", Value::Scalar(in_black / 255.0)), + ("inWhite", Value::Scalar(in_white / 255.0)), + ("gamma", Value::Scalar(*gamma)), + ("outBlack", Value::Scalar(out_black / 255.0)), + ("outWhite", Value::Scalar(out_white / 255.0)), + ], + ), + Adjustment::Exposure { exposure, offset, gamma } => add_effect( + proj, + l, + "Exposure", + comp, + &[("exposure", Value::Scalar(*exposure)), ("offset", Value::Scalar(*offset)), ("gamma", Value::Scalar(*gamma))], + ), + Adjustment::Other(k) => warnings.push(format!("{name}: adjustment `{k}` is not converted (layer kept as an empty adjustment layer)")), + } +} + +// ---------------------------------------------------------------- layer effects → styles + +fn style_mode(m: BlendMode) -> Value { + Value::Enum(STYLE_BLEND_MODES.iter().position(|x| x.1 == m).unwrap_or(0) as u32) +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn color4(c: [f64; 3]) -> Value { + Value::Color([c[0], c[1], c[2], 1.0]) +} + +/// Copy the common style settings (`Md `, `Clr `, `Opct`, angle, distance, size…). +fn common(g: &mut PropGroup, d: &Descriptor) { + if let Some(m) = desc_blend(d.enum_value("Md ")) { + set(g, "blendMode", style_mode(m)); + } + if let Some(c) = d.color("Clr ") { + set(g, "color", color4(c)); + } + for (k, prop) in [("Opct", "opacity"), ("Nose", "noise"), ("Dstn", "distance"), ("blur", "size"), ("lagl", "angle"), ("ShdN", "jitter"), ("Inpr", "range")] + { + if let Some(v) = d.num(k) { + set(g, prop, Value::Scalar(v)); + } + } + if let Some(b) = d.bool("uglg") { + set(g, "useGlobalLight", Value::Bool(b)); + } +} + +fn gradient_of(d: &Descriptor) -> Option { + let g = d.obj("Grad")?; + let list = |k: &str| match g.get(k) { + Some(DValue::List(v)) => v.clone(), + _ => vec![], + }; + let colors: Vec<(f64, [f32; 4])> = list("Clrs") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => { + let col = s.color("Clr ")?; + Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, [col[0] as f32, col[1] as f32, col[2] as f32, 1.0])) + } + _ => None, + }) + .collect(); + let opacities: Vec<(f64, f32)> = list("Trns") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, (s.num("Opct").unwrap_or(100.0) / 100.0) as f32)), + _ => None, + }) + .collect(); + (!colors.is_empty()).then(|| Gradient { colors, opacities: if opacities.is_empty() { vec![(0.0, 1.0), (1.0, 1.0)] } else { opacities } }) +} + +/// The effect descriptor for `key`, or the first enabled one of its `…Multi` list. +fn effect<'a>(fx: &'a Descriptor, key: &str, multi: &str) -> Option<&'a Descriptor> { + fx.obj(key).or_else(|| match fx.get(multi) { + Some(DValue::List(v)) => v.iter().find_map(|x| match x { + DValue::Descriptor(d) if d.bool("enab") != Some(false) => Some(d), + _ => None, + }), + _ => None, + }) +} + +fn apply_effects(proj: &mut Project, layer: &mut Layer, fx: &Descriptor, gl: &styles::GlobalLight) { + const MAP: &[(&str, &str, &str)] = &[ + ("DrSh", "dropShadowMulti", "dropShadow"), + ("IrSh", "innerShadowMulti", "innerShadow"), + ("OrGl", "outerGlowMulti", "outerGlow"), + ("IrGl", "innerGlowMulti", "innerGlow"), + ("ebbl", "bevelEmbossMulti", "bevelEmboss"), + ("ChFX", "satinMulti", "satin"), + ("SoFi", "solidFillMulti", "colorOverlay"), + ("GrFl", "gradientFillMulti", "gradientOverlay"), + ("FrFX", "frameFXMulti", "stroke"), + ]; + for (key, multi, id) in MAP { + let Some(d) = effect(fx, key, multi) else { continue }; + let enabled = d.bool("enab").unwrap_or(true); + let mut ids = Ids(&mut proj.next_id); + let Some(uid) = styles::add_style(layer, &mut ids, id, gl, enabled) else { continue }; + let Some(g) = layer.props.find_group_mut(uid) else { continue }; + common(g, d); + match *id { + "dropShadow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + if let Some(b) = d.bool("layerConceals") { + set(g, "knocksOut", Value::Bool(b)); + } + } + "innerShadow" | "innerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "choke", Value::Scalar(v)); + } + if d.enum_value("glwS") == Some("SrcC") { + set(g, "source", Value::Enum(0)); + } + } + "outerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + } + "bevelEmboss" => { + let style = match d.enum_value("bvlS") { + Some("OtrB") => 0, + Some("Embs") => 2, + Some("PlEb") => 3, + Some("strokeEmboss") => 4, + _ => 1, + }; + set(g, "style", Value::Enum(style)); + let tech = match d.enum_value("bvlT") { + Some("PrBL") => 1, + Some("Slmt") => 2, + _ => 0, + }; + set(g, "technique", Value::Enum(tech)); + if let Some(v) = d.num("srgR") { + set(g, "depth", Value::Scalar(v)); + } + set(g, "direction", Value::Enum(u32::from(d.enum_value("bvlD") == Some("Out ")))); + if let Some(v) = d.num("Sftn") { + set(g, "soften", Value::Scalar(v)); + } + if let Some(v) = d.num("Lald") { + set(g, "altitude", Value::Scalar(v)); + } + for (k, p) in [("hglM", "highlightMode"), ("sdwM", "shadowMode")] { + if let Some(m) = desc_blend(d.enum_value(k)) { + set(g, p, style_mode(m)); + } + } + for (k, p) in [("hglC", "highlightColor"), ("sdwC", "shadowColor")] { + if let Some(c) = d.color(k) { + set(g, p, color4(c)); + } + } + for (k, p) in [("hglO", "highlightOpacity"), ("sdwO", "shadowOpacity")] { + if let Some(v) = d.num(k) { + set(g, p, Value::Scalar(v)); + } + } + } + "satin" => { + if let Some(b) = d.bool("Invr") { + set(g, "invert", Value::Bool(b)); + } + } + "gradientOverlay" => { + if let Some(gr) = gradient_of(d) { + set(g, "colors", Value::Gradient(gr)); + } + if let Some(a) = d.num("Angl") { + set(g, "angle", Value::Scalar(a)); + } + let st = match d.enum_value("Type") { + Some("Rdl ") => 1, + Some("Angl") => 2, + Some("Rflc") => 3, + Some("Dmnd") => 4, + _ => 0, + }; + set(g, "style", Value::Enum(st)); + if let Some(b) = d.bool("Rvrs") { + set(g, "reverse", Value::Bool(b)); + } + if let Some(b) = d.bool("Algn") { + set(g, "alignWithLayer", Value::Bool(b)); + } + if let Some(v) = d.num("Scl ") { + set(g, "scale", Value::Scalar(v)); + } + } + "stroke" => { + if let Some(v) = d.num("Sz ") { + set(g, "size", Value::Scalar(v)); + } + let pos = match d.enum_value("Styl") { + Some("InsF") => 1, + Some("CtrF") => 2, + _ => 0, + }; + set(g, "position", Value::Enum(pos)); + } + _ => {} + } + } +} + +/// Layer ▸ Create ▸ Convert to Editable Text: a footage layer showing a Photoshop type layer +/// becomes a text layer (`None` when the layer's source has no text data). +pub fn editable_text(proj: &mut Project, comp: &Comp, layer: &Layer, psd: &Psd, index: usize) -> Option { + let t = psd.layers.get(index)?.text.as_ref()?; + let mut l = text_layer(proj, comp, &layer.name, t); + l.name = layer.name.clone(); + l.blend_mode = layer.blend_mode; + l.switches.video = layer.switches.video; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.start_time = layer.start_time; + if let (Some(o), Some(p)) = (layer.props.prop("transform/opacity"), l.props.prop_mut("transform/opacity")) { + *p = effectcraft_project::Property { uid: p.uid, ..o.clone() }; + } + Some(l) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blend_keys() { + assert_eq!(blend_mode("mul "), BlendMode::Multiply); + assert_eq!(blend_mode("smud"), BlendMode::Exclusion); + assert_eq!(blend_mode("pass"), BlendMode::Normal); + assert_eq!(desc_blend(Some("linearDodge")), Some(BlendMode::LinearDodge)); + assert_eq!(font_names("Inter-Bold"), ("Inter".into(), "Bold".into())); + } +} diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs new file mode 100644 index 000000000..5b2f635e0 --- /dev/null +++ b/crates/engine/src/roto.rs @@ -0,0 +1,569 @@ +//! Roto Brush & Refine Edge runtime: propagation and Freeze jobs, Input Key maintenance and the +//! tool options. +//! +//! Segmentations are derived data in `effectcraft_effects::roto`'s content-keyed cache (see that +//! module). `roto.propagate` fills it at full resolution in the background (or blocking with +//! `wait`), frame by frame outward from the base frame, rendering the effect's input +//! (`Renderer::layer_input`, source → masks → the effects above it). `roto.freeze` runs the same +//! pass and stores the final mattes in the effect (one undo step). [`sync`], run after every +//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made +//! of (the layer's source and timing, its masks and the effects above it) and drops frozen +//! mattes made from other frames. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; +use effectcraft_effects::{Params, flatten_params}; +use effectcraft_keyframe::Value; +use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; + +use crate::tracking::lock; +use crate::{Event, Session}; + +/// Roto Brush / Refine Edge tool options (serde: `roto.options`). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoOptions { + /// Roto Brush tip diameter (layer pixels). + pub diameter: f64, + /// Refine Edge tip diameter (layer pixels). + pub refine_diameter: f64, + /// Layer panel view: `alphaBoundary` (pink outline), `alpha`, `alphaOverlay`, `none`. + pub view: String, + /// Alpha Overlay colour (straight RGB 0–1) and opacity (percent). + pub overlay_color: [f64; 3], + pub overlay_opacity: f64, + /// Alpha Boundary colour. + pub boundary_color: [f64; 3], + /// Start propagating in the background after each stroke. + pub auto_propagate: bool, +} + +impl Default for RotoOptions { + fn default() -> Self { + RotoOptions { + diameter: 20.0, + refine_diameter: 10.0, + view: "alphaBoundary".into(), + overlay_color: [1.0, 0.0, 0.0], + overlay_opacity: 50.0, + boundary_color: [1.0, 0.25, 0.75], + auto_propagate: true, + } + } +} + +pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"]; + +/// What a job does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum RotoTask { + Propagate, + Freeze, +} + +/// Which way to propagate. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Direction { + Forward, + Backward, + #[default] + Both, +} + +impl Direction { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "forward" | "fwd" => Direction::Forward, + "backward" | "back" => Direction::Backward, + "both" | "all" => Direction::Both, + _ => return None, + }) + } +} + +/// Live job progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RotoProgress { + pub task: Option, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, +} + +impl RotoProgress { + pub fn banner(&self) -> String { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + match self.task { + Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + _ => format!("Propagating Roto Brush: {pc}%"), + } + } +} + +#[derive(Default)] +pub struct RotoShared { + pub cancel: AtomicBool, + pub state: Mutex, + frozen: Mutex>, +} + +pub struct RotoJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + pub task: RotoTask, + /// The strokes the job was started with (a Freeze result is dropped when they changed). + strokes: String, +} + +impl RotoJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> RotoProgress { + lock(&self.shared.state).clone() + } +} + +pub(crate) fn is_roto(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.get(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +fn set_str(g: &mut PropGroup, m: &str, v: String) { + if let Some(p) = g.get_mut(m) { + p.value = Value::Str(v); + p.keys.clear(); + } +} + +fn fnv(s: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +/// What the effect's input frames are made of: the layer's source and timing, its masks and the +/// effects above the Roto Brush. +pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { + let mut s = crate::warp::signature(p, comp, layer); + if let Some(m) = layer.masks() { + s.push_str(&format!("{m:?}")); + } + if let Some(fx) = layer.effects() { + for g in fx.groups() { + if g.uid == uid { + break; + } + s.push_str(&format!("{g:?}")); + } + } + format!("{:016x}", fnv(&s)) +} + +/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. +/// Returns the instances whose segmentation changed (to propagate again). +pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut set = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fxg) = l.effects() else { continue }; + for g in fxg.groups().filter(|g| is_roto(g)) { + let key = input_key(after, *cid, l, g.uid); + let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); + if str_of(g, INPUT_KEY) != key { + set.push((*cid, l.id, g.uid, key)); + todo.push((*cid, l.id, g.uid)); + } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, key) in set { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + let had = !str_of(g, INPUT_KEY).is_empty(); + set_str(g, INPUT_KEY, key); + if had { + set_str(g, FROZEN, String::new()); + } + } + } + todo.retain(|(c, l, u)| { + after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + }); + todo +} + +/// Parameters of an instance with static values (strokes and keys are never animated). +pub(crate) fn params_static(g: &PropGroup) -> Params { + flatten_params(g, &mut |p| p.value.clone()) +} + +/// Evaluated parameters of an instance at comp time `t`. +pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup, t: Tick, expr: Option<&dyn ExprHost>) -> Params { + match p.comp(cid) { + Some(c) => { + let ctx = EvalCtx { expr, ..EvalCtx::new(p, cid, c, t) }; + flatten_params(g, &mut |pr| ctx.value(layer, pr)) + } + None => params_static(g), + } +} + +/// The layer's source size (layer pixels). +pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] { + let (w, h) = effectcraft_render::source_size(p, layer); + if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] } +} + +/// First and last layer frame of the layer inside the comp (inclusive). +pub fn frame_limits(comp: &effectcraft_project::Comp, layer: &Layer) -> [i64; 2] { + let fps = comp.frame_rate.as_f64().max(1e-6); + let lo = layer.layer_time(layer.in_point.max(Tick::ZERO)).seconds(); + let hi = layer.layer_time(layer.out_point.min(comp.duration)).seconds(); + let a = fx::frame_of(lo, fps); + let b = (fx::frame_of(hi, fps) - 1).max(a); + [a, b] +} + +/// Comp time of layer frame `f`. +pub(crate) fn comp_time_of(comp: &effectcraft_project::Comp, layer: &Layer, f: i64) -> Tick { + layer.comp_time(comp.frame_rate.tick_of(f)) +} + +/// What the worker needs, detached from the session. +struct Work { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + params: Params, + size: [f64; 2], + fps: f64, + frames: Vec, + task: RotoTask, +} + +/// Renders the effect's input frames for a worker, keeping the last few. +pub(crate) struct FrameSource<'a> { + r: Renderer<'a>, + ctx0: EvalCtx<'a>, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + memo: HashMap>, +} + +impl<'a> FrameSource<'a> { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + project: &'a Project, + footage: &'a dyn FootageSource, + expr: Option<&'a dyn ExprHost>, + cache: &'a LayerCache, + cid: ItemId, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + ) -> FrameSource<'a> { + let mut r = Renderer::new(project, footage, RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = expr; + r.cache = Some(cache); + let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) }; + FrameSource { r, ctx0, comp, layer, index, size, memo: HashMap::new() } + } + + pub(crate) fn get(&mut self, f: i64) -> Option> { + if let Some(i) = self.memo.get(&f) { + return Some(i.clone()); + } + let t = comp_time_of(self.comp, self.layer, f); + let buf = self.r.layer_input(&self.ctx0.at(t), self.layer, self.index)?; + let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + effectcraft_effects::Buf { + img: effectcraft_raster::resample(&buf.img, w, h), + offset: [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale], + scale: 1.0, + } + } else { + (*buf).clone() + }; + let img = Arc::new(fx::layer_grid(&buf, self.size)); + if self.memo.len() > 4 { + self.memo.clear(); + } + self.memo.insert(f, img.clone()); + Some(img) + } +} + +fn run_work(w: Work, shared: &RotoShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut src = FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size); + { + let mut s = lock(&shared.state); + s.task = Some(w.task); + s.total = w.frames.len() as u64; + } + let cancel = || shared.cancel.load(Ordering::Relaxed); + let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); + let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + for (k, f) in w.frames.iter().enumerate() { + if cancel() { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + return; + } + let mut get = |g: i64| src.get(g); + let ok = match w.task { + RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), + RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + Some((a, b)) => { + if let Some(img) = src.get(*f) { + frozen.size = [img.width as usize, img.height as usize]; + } + frozen.frames.insert(*f, fx::freeze_frame(&a, &b)); + true + } + None => false, + }, + }; + if !ok && !cancel() { + return fail(&format!("frame {f} could not be segmented")); + } + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + if w.task == RotoTask::Freeze { + *lock(&shared.frozen) = Some(frozen); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// Frames to compute, outward from the base frame. +pub(crate) fn job_frames(d: &effectcraft_track::roto::RotoData, dir: Direction) -> Vec { + let Some(base) = d.base else { return vec![] }; + let mut v = vec![base]; + if dir != Direction::Backward { + v.extend(base + 1..=d.span[1]); + } + if dir != Direction::Forward { + v.extend((d.span[0]..base).rev()); + } + v +} + +impl Session { + pub fn is_roto_running(&self) -> bool { + self.roto_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + pub fn roto_progress(&self) -> Option { + self.roto_job.as_ref().map(|j| j.progress()) + } + + /// (comp, layer, effect uid, task) of the running job. + pub fn roto_target(&self) -> Option<(ItemId, LayerId, Uid, RotoTask)> { + self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)) + } + + pub fn stop_roto(&mut self) -> bool { + match &self.roto_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start propagating (or freezing) Roto Brush `effect` on `layer`. `wait`: block until done + /// (wasm32 always does). Returns the number of frames. + pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result { + if self.is_roto_running() { + if !wait { + return Err("a Roto Brush job is already running".into()); + } + self.stop_roto(); + if let Some(mut j) = self.roto_job.take() + && let Some(t) = j.thread.take() + { + let _ = t.join(); + } + } + self.poll_roto_job(); + self.roto_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fxg = l.effects().ok_or("the layer has no effects")?; + let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let size = layer_size(&self.project, c, l); + let fps = c.frame_rate.as_f64(); + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let n = frames.len(); + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + params, + size, + fps, + frames, + task, + }; + let strokes = str_of(g, fx::STROKES); + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(RotoShared::default()); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("roto".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_roto_job(); + Ok(n) + } + + /// Finish a done job: Freeze writes the frozen mattes (one undo step). + fn poll_roto_job(&mut self) -> bool { + let Some(job) = &self.roto_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.roto_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + if let Some(e) = &st.error { + self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.bump(); + return true; + } + if st.cancelled { + self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.bump(); + return true; + } + match job.task { + RotoTask::Propagate => { + self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + // Rendered frames depend on the cache: redraw. + self.bump(); + } + RotoTask::Freeze => { + let Some(fc) = lock(&job.shared.frozen).take() else { return true }; + let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let r = self.edit("Freeze", None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; + if str_of(g, fx::STROKES) != strokes { + return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + } + set_str(g, FROZEN, fc.to_json()); + Ok(()) + }); + let msg = match r { + Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("Roto Brush: {e}"), + }; + self.events.push(Event::Toast { message: msg, error: false }); + } + } + true + } + + /// Frontends call this every frame: finishes jobs and (with `auto`) propagates edited + /// instances in the background. + pub fn poll_roto(&mut self, auto: bool) -> bool { + let changed = self.poll_roto_job(); + if auto && self.roto_job.is_none() && self.state.roto.auto_propagate { + while let Some((c, l, e)) = self.roto_pending.first().copied() { + self.roto_pending.remove(0); + if self.start_roto(c, l, e, RotoTask::Propagate, Direction::Both, false).is_ok() { + return true; + } + } + } + changed + } + + /// The computed frames of an instance (full resolution), by layer frame. + pub fn roto_computed(&self, comp: ItemId, layer: LayerId, effect: Uid) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(c) = self.project.comp(comp) else { return out }; + let Some(l) = c.layer(layer) else { return out }; + let Some(g) = l.props.find_group(effect) else { return out }; + let ps = params_static(g); + let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); + for f in ch.keys.keys() { + out.insert(*f, ch.is_cached(*f, 1.0)); + } + out + } +} diff --git a/crates/engine/src/rq_tests.rs b/crates/engine/src/rq_tests.rs index c4aa08b8a..f172b5e97 100644 --- a/crates/engine/src/rq_tests.rs +++ b/crates/engine/src/rq_tests.rs @@ -159,6 +159,6 @@ fn without_exporter() { s.execute("renderQueue.add", json!({})).unwrap(); assert!(!s.is_enabled("renderQueue.render")); let f = s.execute("renderQueue.formats", json!({})).unwrap(); - assert_eq!(f["formats"].as_array().unwrap().len(), 7); + assert_eq!(f["formats"].as_array().unwrap().len(), 10); assert_eq!(f["formats"][0]["available"], false); } diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index b0512a6a2..e54eb93bb 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -27,6 +27,7 @@ impl Session { self.prefs.normalize(); self.layer_cache.set_budget(self.prefs.layer_cache_bytes()); self.footage.set_cache_budget(self.prefs.media_cache_bytes()); + self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; if self.history.undo.len() > levels { @@ -36,6 +37,42 @@ impl Session { self.prefs_revision += 1; } + /// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache + /// and attach it to the layer cache. Without a folder setting the platform cache folder is + /// used, but only by frontends with a settings store (headless sessions stay off disk). + pub fn configure_disk_cache(&mut self) { + use effectcraft_render::disk_cache::{DiskCache, default_folder, footage_salt}; + let d = &self.prefs.disk; + let folder = if d.disk_cache_folder.trim().is_empty() { + self.config.is_some().then(default_folder) + } else { + Some(std::path::PathBuf::from(d.disk_cache_folder.trim())) + }; + let max = d.disk_cache_max_gb.max(1) as u64 * (1 << 30); + let Some(folder) = folder.filter(|_| d.disk_cache_enabled) else { + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + }; + let dc = match &self.disk_cache { + Some(c) if c.folder() == folder.as_path() => { + c.set_max_bytes(max); + c.clone() + } + _ => match DiskCache::open(&folder, max) { + Ok(c) => c, + Err(e) => { + log::warn!("disk cache at {}: {e}", folder.display()); + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + } + }, + }; + self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + self.disk_cache = Some(dc); + } + /// Write settings to the config store. pub fn save_prefs(&self) { if let Some(c) = &self.config diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 000000000..5bd5539c1 --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_disk_cache.rs b/crates/engine/src/tests_disk_cache.rs new file mode 100644 index 000000000..beaeeadab --- /dev/null +++ b/crates/engine/src/tests_disk_cache.rs @@ -0,0 +1,122 @@ +//! The persistent disk cache (PRV-2) through the session: settings, layer cache persistence +//! across restarts, invalidation on edits, purging. + +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn folder(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-session-disk-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d.to_string_lossy().to_string() +} + +fn session(dir: &str) -> Session { + let mut s = Session::default(); + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir, "disk.diskCacheEnabled": true}})).unwrap(); + // Write every layer buffer, however quick to render. + s.disk_cache.as_ref().unwrap().set_min_layer_ms(0); + s +} + +fn project(s: &mut Session) -> effectcraft_project::ItemId { + s.execute("comp.new", json!({"name": "D", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap(); + s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap(); + s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap(); + s.active_comp_id().unwrap() +} + +#[test] +fn disk_cache_settings_persistence_and_invalidation() { + let dir = folder("persist"); + let mut s = session(&dir); + let st = s.execute_checked("cache.diskStats", json!({})).unwrap(); + assert_eq!(st["enabled"], json!(true)); + assert_eq!(st["maxBytes"], json!(100u64 << 30)); + let cid = project(&mut s); + let t = s.time(); + let first = s.render(cid, t, RenderOpts::default()); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + let written = dc.stats().entries; + assert!(written > 0, "layer buffers written"); + // "Restart": a new session on the same folder and project renders from disk. + let saved = s.project.clone(); + drop(s); + let mut s2 = session(&dir); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + let dc2 = s2.disk_cache.clone().unwrap(); + assert_eq!(dc2.stats().entries, written); + let again = s2.render(cid, t, RenderOpts::default()); + assert!(dc2.stats().hits > 0, "{:?}", dc2.stats()); + assert_eq!(again.data, first.data, "disk-cached pixels are exact"); + // An edit changes the keys: a miss, and new entries. + let hits = dc2.stats().hits; + s2.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 20})).unwrap(); + let edited = s2.render(cid, t, RenderOpts::default()); + assert_ne!(edited.data, first.data); + assert_eq!(dc2.stats().hits, hits); + dc2.flush(); + assert!(dc2.stats().entries > written); + // Purge ▸ All Disk Cache empties it; memory purge leaves the disk alone. + let r = s2.execute_checked("edit.purge", json!({"what": "memory"})).unwrap(); + assert_eq!(r["diskEntries"], json!(0)); + assert!(dc2.stats().entries > 0); + let r = s2.execute_checked("edit.purge", json!({"what": "disk"})).unwrap(); + assert!(r["diskEntries"].as_u64().unwrap() > 0); + assert_eq!(dc2.stats().entries, 0); + // Disabling detaches it. + s2.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap(); + assert!(s2.disk_cache.is_none()); + assert!(s2.layer_cache.disk().is_none()); +} + +#[test] +fn footage_changes_change_the_salt() { + let dir = folder("salt"); + let mut s = session(&dir); + let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id())); + std::fs::write(&foot, b"one").unwrap(); + s.edit("add", None, |p, _| { + let f = effectcraft_project::Footage { + path: foot.to_string_lossy().to_string(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + pixel_aspect: 1.0, + frame_rate: effectcraft_time::FrameRate::FPS_30, + native_rate: None, + duration: effectcraft_time::Tick::ZERO, + has_video: true, + has_audio: false, + alpha: effectcraft_project::AlphaMode::Ignore, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PNG".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: None, + ..Default::default() + }; + p.add_item("f.png", effectcraft_color::Label::Lavender, None, effectcraft_project::ItemKind::Footage(f)); + Ok(()) + }) + .unwrap(); + let a = effectcraft_render::disk_cache::footage_salt(&s.project); + std::fs::write(&foot, b"two, longer").unwrap(); + let b = effectcraft_render::disk_cache::footage_salt(&s.project); + assert_ne!(a, b); + let cid = project(&mut s); + let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap(); + let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + assert_ne!(k1, k2); + // Unrelated comps do not change it. + s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap(); + assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); +} diff --git a/crates/engine/src/tests_project_items.rs b/crates/engine/src/tests_project_items.rs index 4d097aca4..4ab9ce2b2 100644 --- a/crates/engine/src/tests_project_items.rs +++ b/crates/engine/src/tests_project_items.rs @@ -39,3 +39,60 @@ fn move_into_folder_rename_label_comment_with_undo() { s.execute("edit.undo", json!({})).unwrap(); assert_eq!(s.project.item(comp).unwrap().comment, ""); } + +#[test] +fn delete_and_duplicate_items() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Inner", "width": 64, "height": 64, "duration": 1})).unwrap(); + let inner = s.active_comp_id().unwrap(); + s.execute("layer.newNull", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Outer", "width": 64, "height": 64, "duration": 1})).unwrap(); + let outer = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + s.execute("renderQueue.add", json!({"comp": inner.0})).unwrap(); + // Duplicate: a new comp with fresh layer ids. + let r = s.execute("project.duplicate", json!({"items": [inner.0]})).unwrap(); + let dup = ItemId(r["items"][0].as_u64().unwrap()); + assert_eq!(s.project.item(dup).unwrap().name, "Inner 2"); + let a = s.project.comp(inner).unwrap().layers[0].id; + let b = s.project.comp(dup).unwrap().layers[0].id; + assert_ne!(a, b); + // Delete: layers that use the item and its render items go too; undoable. + s.execute("project.delete", json!({"items": ["Inner"]})).unwrap(); + assert!(s.project.item(inner).is_none()); + assert!(s.project.comp(outer).unwrap().layers.is_empty()); + assert!(s.project.render_queue.is_empty()); + s.undo(); + assert!(s.project.item(inner).is_some()); + assert_eq!(s.project.comp(outer).unwrap().layers.len(), 1); + // Folders take their contents. + let f = ItemId(s.execute("project.newFolder", json!({"name": "F"})).unwrap()["item"].as_u64().unwrap()); + s.execute("project.move", json!({"items": [dup.0], "folder": f.0})).unwrap(); + s.execute("project.delete", json!({"items": [f.0]})).unwrap(); + assert!(s.project.item(dup).is_none()); +} + +#[test] +fn property_group_edits() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "C", "width": 64, "height": 64, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap(); + let a = s.execute("effect.apply", json!({"layer": l, "effect": "Gaussian Blur"})).unwrap()["effects"][0].as_u64().unwrap(); + let b = s.execute("effect.apply", json!({"layer": l, "effect": "Invert"})).unwrap()["effects"][0].as_u64().unwrap(); + let fx = |s: &Session| -> Vec<(u64, String, bool)> { + s.active_comp().unwrap().layers[0].props.sub("effects").unwrap().groups().map(|g| (g.uid, g.name.clone(), g.enabled)).collect() + }; + s.execute("prop.renameGroup", json!({"layer": l, "prop": a, "name": "Soft"})).unwrap(); + s.execute("prop.setGroupEnabled", json!({"layer": l, "prop": a, "value": false})).unwrap(); + s.execute("prop.moveGroup", json!({"layer": l, "prop": b, "index": 1})).unwrap(); + let d = s.execute("prop.duplicateGroup", json!({"layer": l, "prop": a})).unwrap()["prop"].as_u64().unwrap(); + let v = fx(&s); + assert_eq!(v.iter().map(|x| x.0).collect::>(), [b, a, d]); + assert_eq!(v[1].1, "Soft"); + assert!(!v[1].2); + s.execute("prop.removeGroup", json!({"layer": l, "prop": a})).unwrap(); + assert_eq!(fx(&s).len(), 2); + // Fixed groups can't be removed. + let tr = s.active_comp().unwrap().layers[0].props.sub("transform").unwrap().uid; + assert!(s.execute("prop.removeGroup", json!({"layer": l, "prop": tr})).is_err()); +} diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs new file mode 100644 index 000000000..794036269 --- /dev/null +++ b/crates/engine/src/tests_roto.rs @@ -0,0 +1,288 @@ +//! Roto Brush & Refine Edge end to end through the commands, on synthetic footage: a textured +//! disk moving over a textured background. + +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::RenderOpts; +use effectcraft_track::roto::rle; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup}; + +const R: f64 = 40.0; + +fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +/// Disk centre on frame `f`. +fn center(f: u32) -> [f64; 2] { + let t = f as f64; + [90.0 + 4.0 * t, 110.0 + 1.5 * t + 5.0 * (t * 0.3).sin()] +} + +/// Disk centre with a jump at frame 8 (propagation loses it there). +fn jumping(f: u32) -> [f64; 2] { + if f < 8 { center(f) } else { [230.0 + 2.0 * (f - 8) as f64, 150.0] } +} + +fn draw(c: [f64; 2]) -> Image { + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let n = noise(p[0] - c[0], p[1] - c[1], 11); + let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32]; + let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9)); + let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + } + } + img +} + +fn gt(c: [f64; 2]) -> String { + let m: Vec = (0..W * H).map(|i| (((i % W) as f64 + 0.5 - c[0]).hypot((i / W) as f64 + 0.5 - c[1]) <= R) as u8).collect(); + rle::encode(&m) +} + +fn line(a: [f64; 2], b: [f64; 2]) -> Value { + json!((0..=8).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 8.0, a[1] + (b[1] - a[1]) * k as f64 / 8.0]).collect::>()) +} + +/// A session with the clip (the Track solid removed) and foreground / background strokes on +/// frame 0. `name`: the footage file name (segmentations are cached process-wide by content key, +/// which includes the footage's identity: different clips need different names). +fn painted(path: fn(u32) -> [f64; 2], name: &str) -> (Session, LayerId, u64) { + let (mut s, clip, solid) = setup(move |f| draw(path(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = name.into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = path(0); + let r = + s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": line([c[0] - 22.0, c[1] - 8.0], [c[0] + 22.0, c[1] + 8.0]), "radius": 6})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert_eq!(r["base"], 0); + assert_eq!(r["span"], json!([0, 20])); + for (a, b) in [([c[0] - 60.0, c[1] - 58.0], [c[0] + 70.0, c[1] - 58.0]), ([c[0] + 70.0, c[1] + 58.0], [c[0] - 60.0, c[1] + 58.0])] { + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line(a, b), "radius": 6})).unwrap(); + } + (s, clip, uid) +} + +/// Drop an instance's derived segmentations from the process-wide cache. +fn forget(s: &Session, clip: LayerId, uid: u64) { + let comp = s.active_comp().unwrap(); + let l = comp.layer(clip).unwrap(); + let g = l.props.find_group(uid).unwrap(); + let size = crate::roto::layer_size(&s.project, comp, l); + let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0); + effectcraft_effects::roto::forget(ch.keys.values().copied()); +} + +fn iou_at(s: &mut Session, clip: LayerId, f: u32, c: [f64; 2]) -> f64 { + let st = s.execute("roto.status", json!({"layer": clip.0, "frame": f, "compute": true, "compareTo": gt(c)})).unwrap(); + st["iou"].as_f64().unwrap_or_else(|| panic!("no matte on frame {f}: {st}")) +} + +fn alpha_at(s: &Session, f: u32) -> Vec { + let img = s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts { scale: 1.0, ..Default::default() }); + img.data.iter().map(|p| p[3]).collect() +} + +#[test] +fn roto_brush_segments_and_propagates() { + let (mut s, clip, uid) = painted(center, "disk-a.mov"); + let l = s.active_comp().unwrap().layer(clip).unwrap().clone(); + assert_eq!(l.effects().unwrap().groups().filter(|g| crate::roto::is_roto(g)).count(), 1, "one effect for all strokes"); + let j0 = iou_at(&mut s, clip, 0, center(0)); + assert!(j0 > 0.95, "base frame IoU {j0}"); + // Propagate the span (blocking). + let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap(); + assert_eq!(r["frames"], 21, "{r}"); + assert_eq!(r["status"]["computed"].as_array().unwrap().len(), 21); + let mut worst: f64 = 1.0; + for f in 0..=20 { + worst = worst.min(iou_at(&mut s, clip, f, center(f))); + } + assert!(worst > 0.9, "worst IoU over the span {worst}"); + // The render applies the matte; outside the span the layer is transparent. + let a = alpha_at(&s, 12); + let c = center(12); + let mut wrong = 0; + for (i, v) in a.iter().enumerate() { + let d = (((i as u32 % W) as f64 + 0.5 - c[0]).hypot((i as u32 / W) as f64 + 0.5 - c[1])) - R; + if (d < -3.0 && *v < 0.5) || (d > 3.0 && *v > 0.5) { + wrong += 1; + } + } + assert!(wrong < 150, "{wrong} wrong pixels in the render"); + assert!(alpha_at(&s, 23).iter().all(|v| *v == 0.0)); + // Extending the span propagates further. + let r = s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "to": 24, "wait": true})).unwrap(); + assert_eq!(r["status"]["span"], json!([0, 24])); + assert!(iou_at(&mut s, clip, 24, center(24)) > 0.9); + // Undo / redo of strokes and the span. + let n = |s: &Session| { + let st = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().get(effectcraft_effects::roto::STROKES).unwrap().value.clone(); + match st { + effectcraft_keyframe::Value::Str(j) => effectcraft_track::roto::RotoData::from_json(&j), + _ => panic!(), + } + }; + assert_eq!(n(&s).span, [0, 24]); + assert!(s.undo()); + assert_eq!(n(&s).span, [0, 20]); + assert_eq!(n(&s).strokes.len(), 3); + assert!(s.undo()); + assert_eq!(n(&s).strokes.len(), 2); + assert!(s.redo()); + assert!(s.redo()); + assert_eq!(n(&s).strokes.len(), 3); + assert_eq!(n(&s).span, [0, 24]); + // Serde keeps the strokes; the reopened project renders the same matte. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + assert_eq!( + crate::roto::params_static(&g).get("strokes"), + crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()).get("strokes") + ); + // Bad parameters are reported. + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "sideways", "points": [[1, 1]]})).is_err()); + assert!(s.execute_checked("roto.propagate", json!({"layer": clip.0, "bogus": 1})).is_err()); + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": [[1, 1]], "frame": 99})).is_err()); +} + +#[test] +fn correction_strokes_restart_propagation() { + let (mut s, clip, _) = painted(jumping, "disk-jump.mov"); + s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); + assert!(iou_at(&mut s, clip, 6, jumping(6)) > 0.9); + let bad = iou_at(&mut s, clip, 8, jumping(8)); + assert!(bad < 0.5, "the jump should break propagation ({bad})"); + // Correct frame 8: paint the disk in and its old place out. + let (c, old) = (jumping(8), center(8)); + s.execute("roto.stroke", json!({"layer": clip.0, "frame": 8, "points": line([c[0] - 20.0, c[1]], [c[0] + 20.0, c[1]]), "radius": 6})).unwrap(); + s.execute( + "roto.stroke", + json!({"layer": clip.0, "frame": 8, "kind": "bg", "points": line([old[0] - 15.0, old[1] - 15.0], [old[0] + 15.0, old[1] + 15.0]), "radius": 8}), + ) + .unwrap(); + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + let computed: Vec = serde_json::from_value(st["computed"].clone()).unwrap(); + assert!(computed.contains(&7) && !computed.contains(&8) && !computed.contains(&9), "frames from the correction on are recomputed: {computed:?}"); + assert_eq!(st["strokeFrames"], json!([0, 8])); + s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "wait": true})).unwrap(); + for f in 8..=20 { + let j = iou_at(&mut s, clip, f, jumping(f)); + assert!(j > 0.9, "frame {f}: IoU {j}"); + } + // Clearing the correction brings the bad frame back. + let r = s.execute("roto.clearStrokes", json!({"layer": clip.0, "frame": 8})).unwrap(); + assert_eq!(r["removed"], 2); + assert!(iou_at(&mut s, clip, 8, jumping(8)) < 0.5); +} + +#[test] +fn freeze_caches_the_mattes() { + let (mut s, clip, uid) = painted(center, "disk-freeze.mov"); + // A refine band along the edge on the base frame. + let c = center(0); + let ring: Vec<[f64; 2]> = (0..=40).map(|k| k as f64 / 40.0 * std::f64::consts::TAU).map(|a| [c[0] + R * a.cos(), c[1] + R * a.sin()]).collect(); + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "refine", "points": ring, "radius": 5})).unwrap(); + s.execute("roto.span", json!({"layer": clip.0, "end": 3})).unwrap(); + let live: Vec> = (0..=3).map(|f| alpha_at(&s, f)).collect(); + // Freeze in the background and wait for it. + let r = s.execute_checked("roto.freeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 4); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["frozen"], true, "{st}"); + // Frozen renders match the live ones (8-bit alpha), even without the derived segmentations. + forget(&s, clip, uid); + s.layer_cache = Default::default(); + for (f, want) in live.iter().enumerate() { + let got = alpha_at(&s, f as u32); + let worst = got.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(worst <= 1.0 / 255.0 + 1e-4, "frame {f}: {worst}"); + } + assert!(st["computed"].as_array().unwrap().len() >= 4); + // Frozen effects refuse strokes; Unfreeze is undoable. + assert!(s.execute("roto.stroke", json!({"layer": clip.0, "points": [[10, 10]]})).is_err()); + s.execute("roto.unfreeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], false); + assert!(s.undo()); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], true); + // Changing the source frames (the layer's timing) drops the frozen mattes. + let cid = s.active_comp_id().unwrap(); + s.edit("shift", None, |p, _| { + let l = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap(); + l.start_time = frame_time(1); + Ok(()) + }) + .unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0, "effect": uid})).unwrap()["frozen"], false); +} + +#[test] +fn propagation_is_deterministic_and_runs_in_background() { + let (mut s, clip, uid) = painted(center, "disk-bg.mov"); + s.execute("roto.span", json!({"layer": clip.0, "end": 5})).unwrap(); + s.execute("roto.propagate", json!({"layer": clip.0})).unwrap(); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["computed"], json!([0, 1, 2, 3, 4, 5]), "{st}"); + let a = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + // Recomputing from scratch gives the same matte. + forget(&s, clip, uid); + let b = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + assert!(a.is_string()); + assert_eq!(a, b); + // Tool options. + let o = s.execute_checked("roto.options", json!({"diameter": 30, "view": "alphaOverlay"})).unwrap(); + assert_eq!(o["view"], "alphaOverlay"); + assert_eq!(s.state.roto.diameter, 30.0); + assert!(s.execute("roto.options", json!({"view": "sideways"})).is_err()); + // The default radius follows the tool diameter. + let r = s.execute("roto.stroke", json!({"layer": clip.0, "points": [[90, 110]]})).unwrap(); + assert_eq!(r["radius"], 15.0); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 924fb5793..b15259692 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -332,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "keys.audioToKeyframes", "track.camera"] { + for id in ["view.displayColorManagement", "keys.audioToKeyframes"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/engine/src/tests_vector_import.rs b/crates/engine/src/tests_vector_import.rs new file mode 100644 index 000000000..d205630c8 --- /dev/null +++ b/crates/engine/src/tests_vector_import.rs @@ -0,0 +1,237 @@ +//! Photoshop import as a composition (IO-2) and Layer ▸ Create text conversions (SHP-5). + +use effectcraft_color::BlendMode; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{ItemKind, LayerSource, MaskMode}; +use effectcraft_psd::write::*; +use effectcraft_psd::{DValue, Descriptor, Rect}; +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-psd-tests-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +/// A layered document exercising groups, masks, blend modes, text, styles and adjustments. +pub(crate) fn sample_psd() -> Vec { + let mut d = WDoc::new(120, 80); + let shadow = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 0.5]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("uglg", DValue::Bool(false)) + .with("lagl", DValue::UnitFloat("#Ang".into(), 45.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 9.0)) + .with("blur", DValue::UnitFloat("#Pxl".into(), 4.0)), + ), + ); + let stroke = Descriptor::new("null").with( + "FrFX", + DValue::Descriptor( + Descriptor::new("FrFX") + .with("enab", DValue::Bool(true)) + .with("Styl", DValue::Enum("FStl".into(), "InsF".into())) + .with("Md ", DValue::Enum("BlnM".into(), "Nrml".into())) + .with("Opct", DValue::UnitFloat("#Prc".into(), 100.0)) + .with("Sz ", DValue::UnitFloat("#Pxl".into(), 2.0)) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 1.0, 0.0]))), + ), + ); + let mut masked = WLayer::solid("Masked Square", Rect::new(10, 10, 40, 40), [0.0, 0.8, 0.2, 1.0]).with_blend(b"mul "); + masked.mask = Some(WMask { + rect: Rect::new(10, 10, 40, 40), + data: (0..1600).map(|i| if i % 40 < 20 { 1.0 } else { 0.0 }).collect(), + default_color: 0, + disabled: false, + }); + let mut hidden = WLayer::solid("Hidden", Rect::new(0, 0, 10, 10), [1.0, 0.0, 1.0, 1.0]); + hidden.hidden = true; + let mut clipped = WLayer::solid("Clipped", Rect::new(60, 30, 20, 20), [1.0, 1.0, 1.0, 1.0]).with_blend(b"scrn").with_opacity(128); + clipped.clipping = true; + let tri = vec![[[70.0, 10.0]; 3], [[110.0, 10.0]; 3], [[90.0, 40.0]; 3]]; + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 120, 80), [0.2, 0.3, 0.4, 1.0]), + WLayer::group_end(), + masked, + hidden, + WLayer::group("Group A", true, *b"pass"), + WLayer::solid("Base", Rect::new(55, 25, 30, 30), [0.9, 0.1, 0.1, 1.0]).with_block(effects_block(&shadow)), + clipped, + WLayer::solid("Vector Masked", Rect::new(60, 0, 60, 50), [0.1, 0.1, 0.9, 1.0]) + .with_block(vector_mask_block(&[tri], 120, 80, false)) + .with_block(effects_block(&stroke)), + WLayer::empty("Hue Shift", vec![hue_sat_block(40, 10, -5)]), + WLayer::empty("Title", vec![text_block("Hello PSD", [8.0, 70.0], "Inter-Bold", 14.0, [1.0, 1.0, 1.0], 0)]), + ]; + write(&d) +} + +#[test] +fn psd_imports_as_composition() { + let path = tmp("sample.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + assert_eq!(s.state.active_comp, Some(cid)); + assert_eq!(s.project.item(cid).unwrap().name, "sample"); + let c = s.project.comp(cid).unwrap().clone(); + assert_eq!((c.width, c.height), (120, 80)); + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Title", "Hue Shift", "Vector Masked", "Clipped", "Base", "Group A", "Background"]); + // Text layer: editable text, positioned at the type origin. + let title = &c.layers[0]; + assert!(matches!(title.source, LayerSource::Text)); + let Some(KV::Text(doc)) = title.props.prop("text/sourceText").map(|p| p.value.clone()) else { panic!("text") }; + assert_eq!(doc.text, "Hello PSD"); + assert_eq!(doc.size, 14.0); + assert_eq!(doc.font, "Inter"); + assert_eq!(doc.style, "Bold"); + assert_eq!(title.props.prop("transform/position").unwrap().value.as_vec2(), [8.0, 70.0]); + // Adjustment layer with Hue/Saturation. + let hue = &c.layers[1]; + assert!(hue.switches.adjustment); + let fx = hue.effects().unwrap().groups().next().unwrap(); + assert_eq!(fx.get("hue").unwrap().value.as_f64(), 40.0); + assert_eq!(fx.get("saturation").unwrap().value.as_f64(), 10.0); + // Vector mask → mask; Stroke style. + let vm = &c.layers[2]; + let m = vm.masks().unwrap().groups().next().unwrap(); + assert!(matches!(m.kind, effectcraft_project::GroupKind::Mask { mode: MaskMode::Add, .. })); + let Some(KV::Path(p)) = m.get("path").map(|p| p.value.clone()) else { panic!("mask path") }; + assert_eq!(p.vertices.len(), 3); + assert!((p.vertices[1][0] - 110.0).abs() < 1e-3 && (p.vertices[2][1] - 40.0).abs() < 1e-3); + let st = vm.props.group("layerStyles/stroke").unwrap(); + assert_eq!(st.get("size").unwrap().value.as_f64(), 2.0); + assert_eq!(st.get("position").unwrap().value, KV::Enum(1)); + // Clipped → preserve transparency; opacity and blend mode. + let clipped = &c.layers[3]; + assert!(clipped.preserve_transparency); + assert_eq!(clipped.blend_mode, BlendMode::Screen); + assert!((clipped.props.prop("transform/opacity").unwrap().value.as_f64() - 128.0 / 255.0 * 100.0).abs() < 1e-9); + // Drop shadow. + let ds = c.layers[4].props.group("layerStyles/dropShadow").unwrap(); + assert_eq!(ds.get("distance").unwrap().value.as_f64(), 9.0); + assert_eq!(ds.get("angle").unwrap().value.as_f64(), 45.0); + assert_eq!(ds.get("useGlobalLight").unwrap().value, KV::Bool(false)); + assert_eq!(ds.get("opacity").unwrap().value.as_f64(), 60.0); + // Group → collapsed precomp (pass through) with its layers. + let g = &c.layers[5]; + assert!(g.switches.collapse); + let LayerSource::Comp { item } = g.source else { panic!("precomp") }; + let inner = s.project.comp(item).unwrap(); + let inner_names: Vec<&str> = inner.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(inner_names, ["Hidden", "Masked Square"]); + assert!(!inner.layers[0].switches.video); + assert_eq!(inner.layers[1].blend_mode, BlendMode::Multiply); + // Footage items name their Photoshop layer; document-sized footage. + let LayerSource::Footage { item } = inner.layers[1].source else { panic!("footage") }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + let sl = f.layer.as_ref().unwrap(); + assert_eq!((sl.name.as_str(), sl.layer_size), ("Masked Square", false)); + assert_eq!((f.width, f.height), (120, 80)); + assert_eq!(s.project.item(item).unwrap().name, "Masked Square/sample.psd"); + // One undo step removes everything. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.comp(cid).is_none()); +} + +#[test] +fn psd_retain_layer_sizes() { + let path = tmp("retain.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "compositionLayerSizes"})).unwrap(); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + let c = s.project.comp(cid).unwrap(); + let base = c.layer_by_name("Base").unwrap(); + let LayerSource::Footage { item } = base.source else { panic!() }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + assert_eq!((f.width, f.height), (30, 30)); + assert!(f.layer.as_ref().unwrap().layer_size); + assert_eq!(base.props.prop("transform/position").unwrap().value.as_vec2(), [70.0, 40.0]); + assert_eq!(base.props.prop("transform/anchor").unwrap().value.as_vec2(), [15.0, 15.0]); + // Vector mask vertices are in the layer's own space. + let vm = c.layer_by_name("Vector Masked").unwrap(); + let Some(KV::Path(p)) = vm.masks().unwrap().groups().next().unwrap().get("path").map(|p| p.value.clone()) else { panic!() }; + assert!((p.vertices[0][0] - 10.0).abs() < 1e-3, "{:?}", p.vertices); +} + +#[test] +fn import_rejects_unknown_kind() { + let mut s = Session::default(); + assert!(s.execute_checked("file.import", json!({"paths": ["x.psd"], "importAs": "banana"})).is_err()); +} + +fn text_session() -> (Session, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 120, "frameRate": 30, "duration": 2})).unwrap(); + let r = s.execute_checked("layer.newText", json!({"text": "Ab8", "size": 64, "position": [160, 80], "fill": "#ffcc00"})).unwrap(); + (s, r["layer"].as_u64().unwrap()) +} + +fn alpha_diff(a: &effectcraft_raster::Image, b: &effectcraft_raster::Image) -> (f32, f32) { + let mut sum = 0.0; + let mut mass = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + sum += (p[3] - q[3]).abs(); + mass += p[3]; + } + (sum / a.data.len() as f32, mass) +} + +#[test] +fn create_shapes_from_text_matches_text_render() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let opts = RenderOpts::default(); + let text_img = s.render(cid, s.time(), opts); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromText"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(matches!(c.layers[0].source, LayerSource::Shape)); + assert_eq!(c.layers[0].name, "Ab8 Outlines"); + assert!(!c.layer(effectcraft_project::LayerId(lid)).unwrap().switches.video, "source hidden"); + let groups: Vec = c.layers[0].props.sub("contents").unwrap().groups().map(|g| g.name.clone()).collect(); + assert_eq!(groups, ["A", "b", "8"]); + let shape_img = s.render(cid, s.time(), opts); + let (d, mass) = alpha_diff(&text_img, &shape_img); + assert!(mass > 500.0, "text drew something"); + assert!(d < 0.01, "mean alpha difference {d}"); + // Fill colour carried over. + let i = shape_img.data.iter().position(|p| p[3] > 0.99).unwrap(); + let p = shape_img.data[i]; + assert!((p[0] - 1.0).abs() < 0.02 && (p[1] - 0.8).abs() < 0.02 && p[2] < 0.02, "{p:?}"); +} + +#[test] +fn create_masks_from_text() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let text_img = s.render(cid, s.time(), RenderOpts::default()); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "masksFromText"})).unwrap(); + let mid = effectcraft_project::LayerId(r["layers"][0].as_u64().unwrap()); + let c = s.active_comp().unwrap(); + let l = c.layer(mid).unwrap(); + assert!(matches!(l.source, LayerSource::Solid { .. })); + // "A", "b" and "8" have 2, 2 and 3 contours. + assert_eq!(l.masks().unwrap().groups().count(), 7); + let img = s.render(cid, s.time(), RenderOpts::default()); + let (d, _) = alpha_diff(&text_img, &img); + assert!(d < 0.01, "mean alpha difference {d}"); + // Other ops need the right layer type. + assert!(s.execute_checked("layer.create", json!({"op": "shapesFromVector", "layers": [lid]})).is_err()); + assert!(s.execute_checked("layer.create", json!({"op": "nullTracing", "layers": [lid]})).is_err()); +} diff --git a/crates/engine/src/vector.rs b/crates/engine/src/vector.rs new file mode 100644 index 000000000..a3f3e040a --- /dev/null +++ b/crates/engine/src/vector.rs @@ -0,0 +1,250 @@ +//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage → +//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines). + +use effectcraft_keyframe::{Gradient, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, Layer, LayerSource, MaskMode, Project, PropGroup}; +use effectcraft_svg::{Affine, Doc, Geom, GradientKind, Node, Paint}; +use kurbo::{BezPath, Point}; + +fn shape_paths(ids: &mut Ids, path: &BezPath) -> Vec { + effectcraft_path::from_kurbo(path) + .into_iter() + .enumerate() + .map(|(i, sp)| { + let mut g = build::shape_path(ids, sp); + g.name = format!("Path {}", i + 1); + g + }) + .collect() +} + +fn det_scale(m: Affine) -> f64 { + let c = m.as_coeffs(); + (c[0] * c[3] - c[1] * c[2]).abs().sqrt() +} + +/// Axis-aligned scale + translation only (basic shapes stay parametric). +fn axis_aligned(m: Affine) -> Option<(f64, f64, f64, f64)> { + let c = m.as_coeffs(); + (c[1].abs() < 1e-9 && c[2].abs() < 1e-9 && c[0] > 0.0 && c[3] > 0.0).then_some((c[0], c[3], c[4], c[5])) +} + +fn gradient_value(g: &effectcraft_svg::Gradient) -> Gradient { + Gradient { + colors: g.stops.iter().map(|(o, c)| (*o, [c[0] as f32, c[1] as f32, c[2] as f32, 1.0])).collect(), + opacities: g.stops.iter().map(|(o, c)| (*o, c[3] as f32)).collect(), + } +} + +/// Gradient start/end points in shape-layer space (`m`: shape user space → layer space). +fn gradient_points(g: &effectcraft_svg::Gradient, m: Affine, bbox: kurbo::Rect) -> (bool, [f64; 2], [f64; 2]) { + let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY }; + let full = m * unit * g.transform; + let p = |x: f64, y: f64| { + let q = full * Point::new(x, y); + [q.x, q.y] + }; + match g.kind { + GradientKind::Linear { x1, y1, x2, y2 } => (false, p(x1, y1), p(x2, y2)), + GradientKind::Radial { cx, cy, r, .. } => (true, p(cx, cy), p(cx + r, cy)), + } +} + +fn shape_items(ids: &mut Ids, s: &effectcraft_svg::Shape, m: Affine) -> Vec { + use kurbo::Shape as _; + let mut items = vec![]; + // Geometry: parametric when the transform keeps rectangles/ellipses axis-aligned. + match (&s.geom, axis_aligned(m)) { + (Geom::Rect { x, y, w, h, rx, ry }, Some((sx, sy, tx, ty))) if (rx - ry).abs() < 1e-9 && (*rx == 0.0 || (sx - sy).abs() < 1e-9) => { + let (w2, h2) = (w * sx, h * sy); + let pos = [tx + (x + w / 2.0) * sx, ty + (y + h / 2.0) * sy]; + let mut g = build::shape_rect(ids, [w2, h2], pos, rx * sx); + g.name = "Rectangle Path 1".into(); + items.push(g); + } + (Geom::Ellipse { cx, cy, rx, ry }, Some((sx, sy, tx, ty))) => { + items.push(build::shape_ellipse(ids, [2.0 * rx * sx, 2.0 * ry * sy], [tx + cx * sx, ty + cy * sy])); + } + (g, _) => items.extend(shape_paths(ids, &(m * g.to_path()))), + } + let bbox = s.geom.to_path().bounding_box(); + if let Some(st) = &s.stroke { + let width = st.width * det_scale(m); + let mut g = match &st.paint { + Paint::Color(c) => build::shape_stroke(ids, [c[0], c[1], c[2], 1.0], width), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_stroke(ids, radial, a, b, gradient_value(gr), width) + } + }; + set(&mut g, "opacity", Value::Scalar(st.opacity * 100.0)); + set(&mut g, "cap", Value::Enum(st.cap as u32)); + set(&mut g, "join", Value::Enum(st.join as u32)); + set(&mut g, "miter", Value::Scalar(st.miter)); + if let Some((d, off)) = &st.dash + && let Some(dg) = g.sub_mut("dashes") + { + let k = det_scale(m); + set(dg, "dash", Value::Scalar(d.first().copied().unwrap_or(0.0) * k)); + set(dg, "gap", Value::Scalar(d.get(1).or(d.first()).copied().unwrap_or(0.0) * k)); + set(dg, "offset", Value::Scalar(off * k)); + } + items.push(g); + } + if let Some(f) = &s.fill { + let mut g = match f { + Paint::Color(c) => build::shape_fill(ids, [c[0], c[1], c[2], 1.0]), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_fill(ids, radial, a, b, gradient_value(gr)) + } + }; + set(&mut g, "opacity", Value::Scalar(s.fill_opacity * 100.0)); + set(&mut g, "rule", Value::Enum(u32::from(s.fill_rule == effectcraft_svg::FillRule::EvenOdd))); + items.push(g); + } + items +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn group(ids: &mut Ids, name: &str, items: Vec, opacity: f64) -> PropGroup { + let mut g = build::shape_group(ids, name, items); + if let Some(p) = g.prop_mut("transform/opacity") { + p.value = Value::Scalar(opacity * 100.0); + } + g +} + +/// Shape layer contents for an SVG document, in document pixels (transforms are baked into the +/// geometry; group and element opacity go to the group transforms). Top of the paint order first. +pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec { + fn walk(ids: &mut Ids, nodes: &[Node], m: Affine) -> Vec { + let mut out = vec![]; + for n in nodes.iter().rev() { + match n { + Node::Group(g) => { + let items = walk(ids, &g.children, m * g.transform); + if !items.is_empty() { + out.push(group(ids, &g.name, items, g.opacity)); + } + } + Node::Shape(s) => { + let items = shape_items(ids, s, m * s.transform); + if !items.is_empty() { + out.push(group(ids, &s.name, items, s.opacity)); + } + } + } + } + out + } + let root = &doc.root; + let items = walk(ids, &root.children, root.transform); + if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items } +} + +/// A copy of `src`'s transform group with fresh uids. +fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) { + let Some(tr) = src.props.sub("transform") else { return }; + let mut tr = tr.clone(); + let mut n = proj.next_id; + tr.reassign_uids(&mut n); + proj.next_id = n + 1; + if let Some(slot) = dst.props.children.iter_mut().find(|c| c.match_id() == "transform") { + *slot = tr.into(); + } +} + +fn like_source(dst: &mut Layer, src: &Layer) { + dst.start_time = src.start_time; + dst.in_point = src.in_point; + dst.out_point = src.out_point; + dst.stretch = src.stretch; + dst.parent = src.parent; + dst.switches.three_d = src.switches.three_d; +} + +/// Layer ▸ Create ▸ Create Shapes from Vector Layer: `doc` is the SVG shown by `src`. +pub fn shapes_from_vector(proj: &mut Project, comp: &Comp, src: &Layer, doc: &Doc) -> Layer { + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let items = svg_contents(&mut Ids(&mut proj.next_id), doc); + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(items.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + l +} + +/// Layer ▸ Create ▸ Create Shapes from Text: one group per character with its outline, fill and +/// stroke (as laid out at the comp time `ctx` was made for). +pub fn shapes_from_text(proj: &mut Project, comp: &Comp, ctx_project: &Project, cid: ItemId, src: &Layer, time: effectcraft_time::Tick) -> Option { + let ctx = effectcraft_render::EvalCtx::new(ctx_project, cid, comp, time); + let geom = effectcraft_render::text::text_geom(&ctx, src)?; + let chars: Vec = geom.doc.text.chars().filter(|c| !c.is_whitespace()).collect(); + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let mut groups = vec![]; + { + let mut ids = Ids(&mut proj.next_id); + for (i, g) in geom.glyphs.iter().enumerate() { + let path = g.path(); + if path.elements().is_empty() { + continue; + } + let mut items = shape_paths(&mut ids, &path); + if g.stroke_width > 0.0 && g.stroke[3] > 0.0 { + let c = g.stroke; + items.push(build::shape_stroke(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0], g.stroke_width)); + } + if g.apply_fill { + let c = g.fill; + let mut f = build::shape_fill(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0]); + set(&mut f, "opacity", Value::Scalar(c[3] as f64 * 100.0)); + items.push(f); + } + let name = chars.get(i).map(|c| c.to_string()).unwrap_or_else(|| format!("Glyph {}", i + 1)); + groups.push(group(&mut ids, &name, items, 1.0)); + } + } + // First character on top, as in After Effects. + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(groups.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + Some(l) +} + +/// Glyph outlines of a text layer in comp space (for Create Masks from Text). +pub fn text_outlines_in_comp(project: &Project, cid: ItemId, comp: &Comp, src: &Layer, time: effectcraft_time::Tick) -> Vec { + let ctx = effectcraft_render::EvalCtx::new(project, cid, comp, time); + let Some(geom) = effectcraft_render::text::text_geom(&ctx, src) else { return vec![] }; + let (m, _) = ctx.layer_to_comp(src); + let mut out = vec![]; + for g in &geom.glyphs { + let p = g.path(); + let p = effectcraft_path::transform(std::slice::from_ref(&p), &m).remove(0); + out.extend(effectcraft_path::from_kurbo(&p)); + } + out +} + +/// A comp-sized white solid with one mask per glyph contour (Difference, so counters stay open). +pub fn masks_layer(proj: &mut Project, comp: &Comp, name: &str, outlines: Vec, solid: ItemId) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: solid }, (comp.width, comp.height), None); + let mut ids = Ids(&mut proj.next_id); + if let Some(masks) = l.props.sub_mut("masks") { + for (i, sp) in outlines.into_iter().enumerate() { + masks.children.push(build::mask(&mut ids, &format!("Mask {}", i + 1), sp, MaskMode::Difference, [255, 255, 0]).into()); + } + } + l +} diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 34c83aabd..15f9b17d2 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -18,6 +18,8 @@ filmcraft-h264enc = { workspace = true } filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +effectcraft-opusenc = { workspace = true } +effectcraft-vp9enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -28,3 +30,5 @@ log = { workspace = true } [dev-dependencies] effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } +filmcraft-opus = { workspace = true } +filmcraft-vp9 = { workspace = true } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 86568871e..941a77234 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,6 +7,8 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | +//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -18,6 +20,7 @@ mod encode; mod out; +mod webm; use web_time::Instant; @@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< let mut report = match job.output.format { OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs new file mode 100644 index 000000000..eb99cd227 --- /dev/null +++ b/crates/export/src/webm.rs @@ -0,0 +1,368 @@ +//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF. +//! +//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML, +//! the Matroska element registry and the WebM container guidelines): EBML header, Segment +//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are +//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose +//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every +//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit +//! extended sample rate) follow their published file layouts. + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +const OPUS_RATE: u32 = 48_000; +const OPUS_BITRATE: u32 = 192_000; + +// Element ids (with their length markers). +const EBML: u32 = 0x1A45_DFA3; +const SEGMENT: u32 = 0x1853_8067; +const INFO: u32 = 0x1549_A966; +const TRACKS: u32 = 0x1654_AE6B; +const TRACK_ENTRY: u32 = 0xAE; +const CLUSTER: u32 = 0x1F43_B675; + +fn id_bytes(id: u32) -> Vec { + let b = id.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(3); + b[skip..].to_vec() +} + +/// EBML variable-size integer for an element data size (shortest form). +fn vint(n: u64) -> Vec { + for len in 1..=8u32 { + let max = (1u64 << (7 * len)) - 2; + if n <= max { + let marked = n | (1u64 << (7 * len)); + return marked.to_be_bytes()[8 - len as usize..].to_vec(); + } + } + let mut v = vec![0x01]; + v.extend_from_slice(&n.to_be_bytes()[1..]); + v +} + +fn el(out: &mut Vec, id: u32, data: &[u8]) { + out.extend(id_bytes(id)); + out.extend(vint(data.len() as u64)); + out.extend_from_slice(data); +} + +fn el_uint(out: &mut Vec, id: u32, v: u64) { + let b = v.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(7); + el(out, id, &b[skip..]); +} + +fn el_float(out: &mut Vec, id: u32, v: f64) { + el(out, id, &v.to_be_bytes()); +} + +fn el_str(out: &mut Vec, id: u32, s: &str) { + el(out, id, s.as_bytes()); +} + +/// A block payload: track number, timestamp relative to the cluster, flags, frame. +fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { + let mut b = vint(track); + b.extend_from_slice(&rel.to_be_bytes()); + b.push(flags); + b.extend_from_slice(data); + b +} + +struct Frame { + ms: i64, + track: u64, + data: Vec, + alpha: Option>, +} + +fn cluster(frames: &mut [Frame]) -> Vec { + frames.sort_by_key(|f| (f.ms, f.track)); + let base = frames.first().map_or(0, |f| f.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for f in frames.iter() { + let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + match &f.alpha { + None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + Some(a) => { + let mut g = vec![]; + el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + let mut more = vec![]; + el_uint(&mut more, 0xEE, 1); + el(&mut more, 0xA5, a); + let mut adds = vec![]; + el(&mut adds, 0xA6, &more); + el(&mut g, 0x75A1, &adds); + el(&mut body, 0xA0, &g); + } + } + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let alpha = job.output.channels == Channels::Rgba; + let quality = job.output.quality.clamp(1, 100); + let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + // EBML header. + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + // Segment with an 8-byte size patched at the end. + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_VP9"); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + if alpha { + el_uint(&mut v, 0x55EE, 1); + } + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + if alpha { + el_uint(&mut video, 0x53C0, 1); + } + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + // Audio: mixed in blocks, encoded in 20 ms packets. + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + // Pad the last packet (pre-skip + padding cover the encoder delay). + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + while i < total { + let end = (i + batch).min(total); + let encoded: Vec<(Vec, Option>)> = (i..end) + .into_par_iter() + .map(|k| { + let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); + let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); + let color = enc.encode_yuv420(&y, &u, &vv); + let a = alpha.then(|| { + let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); + ea.encode_yuv420(&y, &u, &vv) + }); + (color, a) + }) + .collect(); + let mut frames: Vec = + encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut frames, end >= total); + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + st.advance(end - i)?; + i = end; + } + if total == 0 { + let mut frames = vec![]; + audio_until(span_end, &mut frames, true); + if !frames.is_empty() { + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + } + } + // Patch the Segment size. + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} + +// ---------------------------------------------------------------- audio-only outputs + +fn ieee_extended(v: f64) -> [u8; 10] { + // 80-bit IEEE 754 extended precision (AIFF COMM sample rate). + let mut out = [0u8; 10]; + if v <= 0.0 { + return out; + } + let e = v.log2().floor() as i32; + let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64; + out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. +pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { + let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let (span_start, span_end) = job.settings.span(comp); + let start = span_start.to_units_floor(sr as i64); + let end = span_end.to_units_floor(sr as i64); + let n = (end - start).max(0) as usize; + let mut pcm = Vec::with_capacity(n * 4); + // Mix in one-second blocks, reporting progress in frames. + let block = sr as usize; + let mut done = 0usize; + let total_frames = st.total.max(1); + let mut reported = 0u64; + while done < n { + let k = block.min(n - done); + let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); + for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { + let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; + pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); + } + done += k; + let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); + if now > reported && st.total > 0 { + st.advance(now - reported)?; + reported = now; + } + } + if reported < st.total { + st.advance(st.total - reported)?; + } + let mut out = vec![]; + let frames = (pcm.len() / 4) as u32; + if aiff { + let mut comm = vec![]; + comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&frames.to_be_bytes()); + comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&ieee_extended(sr as f64)); + let ssnd_len = 8 + pcm.len(); + out.extend_from_slice(b"FORM"); + out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes()); + out.extend_from_slice(b"AIFF"); + out.extend_from_slice(b"COMM"); + out.extend_from_slice(&(comm.len() as u32).to_be_bytes()); + out.extend_from_slice(&comm); + out.extend_from_slice(b"SSND"); + out.extend_from_slice(&(ssnd_len as u32).to_be_bytes()); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&pcm); + if ssnd_len % 2 == 1 { + out.push(0); + } + } else { + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&sr.to_le_bytes()); + out.extend_from_slice(&(sr * 4).to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); + out.extend_from_slice(&pcm); + } + let mut file = crate::out::create(job.sink, job.path)?; + file.write_all(&out).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vints_and_ids() { + assert_eq!(vint(0), vec![0x80]); + assert_eq!(vint(126), vec![0xFE]); + assert_eq!(vint(127), vec![0x40, 0x7F]); + assert_eq!(id_bytes(0xA3), vec![0xA3]); + assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]); + assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]); + assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]); + } +} diff --git a/crates/export/tests/opus_roundtrip.rs b/crates/export/tests/opus_roundtrip.rs new file mode 100644 index 000000000..5ea099f06 --- /dev/null +++ b/crates/export/tests/opus_roundtrip.rs @@ -0,0 +1,364 @@ +//! Round trip of `effectcraft-opusenc` through FilmCraft's spec-derived Opus decoder, plus an +//! ffmpeg oracle (skipped when ffmpeg is not installed) reading an Ogg Opus file written here. + +use std::process::Command; + +use effectcraft_opusenc::OpusEncoder; + +const SR: f32 = 48_000.0; +const FRAME: usize = OpusEncoder::FRAME_SIZE; + +/// Encodes interleaved `pcm`, padding the tail so the encoder's look-ahead is flushed. +fn encode(pcm: &[f32], channels: u8, bitrate: u32) -> (OpusEncoder, Vec>) { + let c = channels as usize; + let mut enc = OpusEncoder::new(channels, bitrate); + let total = pcm.len() / c + enc.pre_skip() as usize; + let frames = total.div_ceil(FRAME); + let mut padded = pcm.to_vec(); + padded.resize(frames * FRAME * c, 0.0); + let packets = padded.chunks(FRAME * c).map(|f| enc.encode_float(f)).collect(); + (enc, packets) +} + +/// Decodes with filmcraft-opus, drops the pre-skip and returns planar channels of `len` samples. +fn decode(head: &[u8], packets: &[Vec], len: usize) -> Vec> { + let mut dec = filmcraft_opus::Decoder::new(head).expect("OpusHead"); + dec.set_trim_pre_skip(true); + let mut out: Vec> = vec![Vec::new(); dec.channels()]; + for p in packets { + let planes = dec.decode(Some(p)).expect("decode"); + for (o, p) in out.iter_mut().zip(planes) { + o.extend_from_slice(&p); + } + } + for o in &mut out { + assert!(o.len() >= len, "decoded {} < {len}", o.len()); + o.truncate(len); + } + out +} + +fn roundtrip(planar: &[Vec], bitrate: u32) -> (Vec>, Vec>) { + let c = planar.len(); + let len = planar[0].len(); + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, c as u8, bitrate); + let decoded = decode(&enc.opus_head(), &packets, len); + (packets, decoded) +} + +/// SNR in dB over the steady part (the first and last 20 ms are excluded). +fn snr(orig: &[f32], dec: &[f32]) -> f64 { + let (a, b) = (FRAME, orig.len() - FRAME); + let mut s = 0f64; + let mut e = 0f64; + for i in a..b { + s += (orig[i] as f64).powi(2); + e += (orig[i] as f64 - dec[i] as f64).powi(2); + } + 10.0 * (s / e.max(1e-30)).log10() +} + +fn sine(freq: f32, amp: f32, len: usize) -> Vec { + (0..len).map(|i| amp * (2.0 * std::f32::consts::PI * freq * i as f32 / SR).sin()).collect() +} + +fn chirp(f0: f32, f1: f32, amp: f32, len: usize) -> Vec { + let dur = len as f32 / SR; + let k = (f1 - f0) / dur; + (0..len) + .map(|i| { + let t = i as f32 / SR; + amp * (2.0 * std::f32::consts::PI * (f0 * t + 0.5 * k * t * t)).sin() + }) + .collect() +} + +fn noise(amp: f32, len: usize, seed: u32) -> Vec { + let mut s = seed; + (0..len) + .map(|_| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + amp * ((s >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0) + }) + .collect() +} + +/// Energy (dB) of `x` in each of a few octave bands up to 16 kHz, from a Hann-windowed FFT +/// averaged over 2048-sample blocks. +fn band_energies(x: &[f32]) -> Vec { + const N: usize = 2048; + let edges = [100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0]; + let mut acc = vec![0f64; edges.len() - 1]; + let mut start = N; + while start + N <= x.len() - N { + let mut re: Vec = (0..N).map(|i| x[start + i] as f64 * (0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / N as f64).cos())).collect(); + let mut im = vec![0f64; N]; + fft(&mut re, &mut im); + for k in 1..N / 2 { + let f = k as f64 * SR as f64 / N as f64; + if let Some(b) = edges.windows(2).position(|w| f >= w[0] && f < w[1]) { + acc[b] += re[k] * re[k] + im[k] * im[k]; + } + } + start += N; + } + acc.iter().map(|e| 10.0 * (e + 1e-20).log10()).collect() +} + +fn fft(re: &mut [f64], im: &mut [f64]) { + let n = re.len(); + let mut j = 0; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let ang = -2.0 * std::f64::consts::PI / len as f64; + for s in (0..n).step_by(len) { + for k in 0..len / 2 { + let (wr, wi) = ((ang * k as f64).cos(), (ang * k as f64).sin()); + let (a, b) = (s + k, s + k + len / 2); + let tr = re[b] * wr - im[b] * wi; + let ti = re[b] * wi + im[b] * wr; + re[b] = re[a] - tr; + im[b] = im[a] - ti; + re[a] += tr; + im[a] += ti; + } + } + len <<= 1; + } +} + +#[test] +fn sine_440_mono() { + let x = sine(440.0, 0.5, 48_000); + for rate in [64_000, 128_000] { + let (packets, d) = roundtrip(std::slice::from_ref(&x), rate); + assert!(packets.iter().all(|p| p.len() == rate as usize / 400)); + let s = snr(&x, &d[0]); + eprintln!("sine 440 Hz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 20.0, "sine SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn chirp_mono() { + let x = chirp(100.0, 10_000.0, 0.4, 96_000); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("chirp 100 Hz-10 kHz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "chirp SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn white_noise_mono() { + let x = noise(0.3, 48_000, 12345); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + let (eo, ed) = (band_energies(&x), band_energies(&d[0])); + let err: Vec = x.iter().zip(&d[0]).map(|(a, b)| a - b).collect(); + let ee = band_energies(&err); + let octave_snr: Vec = eo.iter().zip(&ee).map(|(a, b)| a - b).collect(); + let worst = eo.iter().zip(&ed).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max); + let min_octave = octave_snr.iter().copied().fold(f64::MAX, f64::min); + eprintln!( + "white noise mono @ {} kb/s: full-band SNR {s:.1} dB, octave SNRs 100 Hz-16 kHz {:.1?} dB, worst octave energy error {worst:.2} dB", + rate / 1000, + octave_snr + ); + // Full-band noise loses everything above 20 kHz (CELT's last band ends there) and spends + // few bits per coefficient on 16-20 kHz, so the full-band waveform SNR is low; the band + // energies must match closely and the octaves below 16 kHz must be waveform-coded. + assert!(worst < 1.5, "band energy error {worst:.2} dB at {rate}: {eo:?} vs {ed:?}"); + assert!(s > 0.5, "noise SNR {s:.1} dB at {rate}"); + assert!(min_octave > if rate >= 128_000 { 10.0 } else { 3.0 }, "octave SNRs {octave_snr:?} at {rate}"); + } +} + +#[test] +fn lowpass_noise_mono() { + // White noise through a 4-pole low-pass at about 2 kHz: a dense spectrum the encoder can + // afford to code as a waveform. + let w = noise(0.8, 48_000, 999); + let a = (-2.0 * std::f32::consts::PI * 2_000.0 / SR).exp(); + let mut st = [0f32; 4]; + let x: Vec = w + .iter() + .map(|&v| { + let mut y = v; + for s in st.iter_mut() { + *s = (1.0 - a) * y + a * *s; + y = *s; + } + y + }) + .collect(); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("low-pass noise mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "low-pass noise SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn silence_stays_silent() { + let x = vec![0f32; 48_000]; + let (packets, d) = roundtrip(&[x.clone(), x], 128_000); + assert!(packets.iter().all(|p| p.len() == 3), "silence frames are 3-byte packets"); + for ch in &d { + let peak = ch.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(peak < 1e-6, "peak {peak}"); + } +} + +#[test] +fn stereo_distinct_channels() { + let l = sine(440.0, 0.5, 48_000); + let r = chirp(200.0, 6_000.0, 0.3, 48_000); + let (packets, d) = roundtrip(&[l.clone(), r.clone()], 256_000); + assert!(packets.iter().all(|p| p[0] == 0xFC && p.len() == 640)); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 256 kb/s: L (sine) SNR {sl:.1} dB, R (chirp) SNR {sr:.1} dB"); + assert!(sl > 20.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); + // No leakage of the chirp into the left channel (2-4 kHz octave: chirp only). + let (el, er) = (band_energies(&d[0]), band_energies(&r)); + assert!(el[4] < er[4] - 30.0, "chirp energy leaked into L: {} vs {}", el[4], er[4]); +} + +#[test] +fn stereo_128k_total() { + let l = sine(1000.0, 0.4, 48_000); + let r = sine(330.0, 0.4, 48_000); + let (_, d) = roundtrip(&[l.clone(), r.clone()], 128_000); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 128 kb/s: L SNR {sl:.1} dB, R SNR {sr:.1} dB"); + assert!(sl > 15.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); +} + +#[test] +fn signal_after_silence_and_loud_transient() { + // Silence, then a burst at full scale, then silence again: energy prediction must recover. + let mut x = vec![0f32; 24_000]; + x.extend(sine(2_000.0, 0.99, 24_000)); + x.extend(vec![0f32; 24_000]); + let (_, d) = roundtrip(std::slice::from_ref(&x), 96_000); + let s = snr(&x[24_000 + 2 * FRAME..48_000], &d[0][24_000 + 2 * FRAME..48_000]); + eprintln!("burst after silence @ 96 kb/s: SNR {s:.1} dB"); + assert!(s > 15.0, "burst SNR {s:.1}"); + let tail_peak = d[0][48_000 + 2 * FRAME..].iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(tail_peak < 1e-3, "tail peak {tail_peak}"); +} + +// ---- Ogg Opus (RFC 3533 + RFC 7845) for the ffmpeg oracle ---- + +fn ogg_crc(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + crc ^= (b as u32) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} + +fn ogg_page(out: &mut Vec, packet: &[u8], header_type: u8, granule: u64, seq: u32) { + let mut lacing = vec![255u8; packet.len() / 255]; + lacing.push((packet.len() % 255) as u8); + let start = out.len(); + out.extend_from_slice(b"OggS"); + out.push(0); + out.push(header_type); + out.extend_from_slice(&granule.to_le_bytes()); + out.extend_from_slice(&0x4543_4F50u32.to_le_bytes()); // serial + out.extend_from_slice(&seq.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); // CRC placeholder + out.push(lacing.len() as u8); + out.extend_from_slice(&lacing); + out.extend_from_slice(packet); + let crc = ogg_crc(&out[start..]); + out[start + 22..start + 26].copy_from_slice(&crc.to_le_bytes()); +} + +fn ogg_opus(enc: &OpusEncoder, packets: &[Vec], len: usize) -> Vec { + let mut out = Vec::new(); + ogg_page(&mut out, &enc.opus_head(), 0x02, 0, 0); + let vendor = b"effectcraft-opusenc"; + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes()); + tags.extend_from_slice(vendor); + tags.extend_from_slice(&0u32.to_le_bytes()); + ogg_page(&mut out, &tags, 0, 0, 1); + let pre_skip = enc.pre_skip() as u64; + for (i, p) in packets.iter().enumerate() { + let last = i + 1 == packets.len(); + let granule = if last { pre_skip + len as u64 } else { ((i + 1) * FRAME) as u64 }; + ogg_page(&mut out, p, if last { 0x04 } else { 0 }, granule, i as u32 + 2); + } + out +} + +#[test] +fn ogg_crc_known_value() { + // CRC-32 (poly 0x04C11DB7, no reflection, init 0) of "123456789". + assert_eq!(ogg_crc(b"123456789"), 0x89A1_897F); +} + +#[test] +fn ffmpeg_oracle_decodes_ogg_opus() { + if Command::new("ffmpeg").arg("-version").output().is_err() { + eprintln!("ffmpeg not installed; skipping the oracle test"); + return; + } + for (channels, rate) in [(1u8, 96_000u32), (2, 192_000)] { + let len = 48_000; + let planar: Vec> = + if channels == 1 { vec![chirp(150.0, 9_000.0, 0.4, len)] } else { vec![sine(440.0, 0.5, len), chirp(150.0, 9_000.0, 0.3, len)] }; + let c = channels as usize; + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, channels, rate); + let path = std::env::temp_dir().join(format!("effectcraft-opusenc-{}-{channels}.opus", std::process::id())); + std::fs::write(&path, ogg_opus(&enc, &packets, len)).unwrap(); + let out = Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "f32le", "-ac", &channels.to_string(), "-ar", "48000", "-"]) + .output() + .expect("run ffmpeg"); + let _ = std::fs::remove_file(&path); + assert!(out.status.success(), "ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr)); + let dec: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + assert_eq!(dec.len(), len * c, "ffmpeg must honour pre-skip and the end granule"); + for ch in 0..c { + let d: Vec = (0..len).map(|i| dec[i * c + ch]).collect(); + let s = snr(&planar[ch], &d); + eprintln!("ffmpeg {channels} ch @ {} kb/s, channel {ch}: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "ffmpeg-decoded SNR {s:.1} dB (channel {ch})"); + } + } +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 186aa1a97..1ade594f8 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -363,3 +363,95 @@ fn movies_carry_audio() { assert!(!effectcraft_media::probe(&path).expect("probe").has_audio); } } + +#[test] +fn webm_vp9_opus_roundtrip() { + let d = out_dir("webm"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("comp.webm"); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio && r.bytes > 200, "{r:?}"); + // Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders). + let f = decode_movie(&path, FRAMES, 0.08); + assert!(f.has_audio); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + let l = rms(s.iter().step_by(2).copied()); + assert!((l - 0.3536).abs() < 0.06, "left RMS {l}"); + ffmpeg_check(&path, 0.08); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("vp9|64|48|yuv420p"), "{s}"); + assert!(s.contains("opus"), "{s}"); + } +} + +#[test] +fn webm_with_alpha() { + let d = out_dir("webm-alpha"); + let (p, cid) = project(None); + // Transparent outside the red solid. + let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("alpha.webm"); + run(&p, cid, &NoFootage, &om, &path); + let f = effectcraft_media::probe(&path).expect("probe our output"); + assert_eq!((f.width, f.height), (W, H)); + let bytes = std::fs::read(&path).unwrap(); + // AlphaMode element (0x53C0) and BlockAdditions (0x75A1). + assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1"); + assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions"); + // ffmpeg's libvpx decoder reads the alpha channel. + let png = path.with_extension("alpha.png"); + let ok = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"]) + .arg(&path) + .args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"]) + .arg(&png) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + let img = image::open(&png).unwrap().to_rgba8(); + assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent"); + assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque"); + } else { + eprintln!("ffmpeg with libvpx not available: skipping alpha oracle"); + } +} + +#[test] +fn wav_and_aiff_audio_only() { + let d = out_dir("audio-only"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] { + let om = OutputModule::for_format(fmt); + let path = d.join(name); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..4], magic); + // 1.2 s of 48 kHz 16-bit stereo. + assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len()); + let l = if fmt == OutputFormat::Wav { + // Decoded by FilmCraft. + let f = effectcraft_media::probe(&path).expect("probe audio"); + assert!(f.has_audio && !f.has_video, "{name}"); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + rms(s.iter().step_by(2).copied()) + } else { + // AIFF: big-endian samples after the SSND chunk's offset/block size. + let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16; + let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0); + rms(left) + }; + assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}"); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("pcm_s16"), "{s}"); + } + } +} diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs new file mode 100644 index 000000000..8156d8505 --- /dev/null +++ b/crates/export/tests/vp9_roundtrip.rs @@ -0,0 +1,149 @@ +//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and +//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR. + +use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder}; +use filmcraft_vp9::{Decoder, Plane}; + +fn plane_u8(p: &Plane) -> Vec { + match p { + Plane::U8(v) => v.clone(), + Plane::U16(v) => v.iter().map(|x| *x as u8).collect(), + } +} + +/// Synthetic planes: gradients, noise and sharp edges. +fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut s = 99u32 + kind; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 24) as u8 + }; + let y: Vec = (0..w * h) + .map(|i| { + let (x, yy) = (i % w, i / w); + match kind { + 0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2, + 1 => rnd(), + _ => { + if (x / 7 + yy / 5) % 2 == 0 { + 30 + } else { + 220 + } + } + } + }) + .collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect(); + (y, u, v) +} + +fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { + let mut d = Decoder::new(); + let mut pics = d.decode(frame, 0).expect("decode"); + assert_eq!(pics.len(), 1); + pics.remove(0) +} + +fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec { + (0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect() +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::() / a.len() as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +#[test] +fn lossless_is_exact_for_odd_sizes() { + for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { + for kind in 0..3 { + let (y, u, v) = source(w, h, kind); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert_eq!((p.width, p.height), (w as u32, h as u32)); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma"); + assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u"); + assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v"); + } + } +} + +#[test] +fn lossy_quality_and_size() { + let (w, h) = (200, 120); + for kind in [0, 2] { + let (y, u, v) = source(w, h, kind); + let mut last = 0usize; + for q in [30u8, 60, 80, 95] { + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + let got = crop(&plane_u8(&p.y), p.y_stride, w, h); + let db = psnr(&got, &y); + let min = match q { + 30 => 20.0, + 60 => 28.0, + 80 => 32.0, + _ => 38.0, + }; + assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len()); + assert!(f.len() >= last, "higher quality, more bytes"); + last = f.len(); + } + } +} + +#[test] +fn wide_frames_use_tile_columns() { + // 4160 px > 64 superblocks: two tile columns are required. + let (w, h) = (4160, 16); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); +} + +#[test] +fn ffmpeg_decodes_our_stream() { + let (w, h) = (96, 64); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let frame = e.encode_yuv420(&y, &u, &v); + // IVF container (one frame). + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&0u64.to_le_bytes()); + ivf.extend_from_slice(&frame); + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); + assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); +} diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index b43271c70..635e32605 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-host" -description = "Builds a fully wired EffectCraft session: media decoding (FilmCraft codecs), importer and the expression engine" +description = "Builds a fully wired EffectCraft session: media decoding (FilmCraft codecs), importer, the expression engine and scripting" version.workspace = true edition.workspace = true license.workspace = true @@ -14,7 +14,12 @@ effectcraft-engine = { workspace = true } effectcraft-project = { workspace = true } effectcraft-media = { workspace = true } effectcraft-expr = { workspace = true } +effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-time = { workspace = true } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index b90a42f0b..d7668fcc5 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -1,12 +1,14 @@ //! A fully wired [`Session`]: footage decoding through `effectcraft-media` (FilmCraft's codecs), -//! the media importer, the expression engine and Render Queue export through -//! `effectcraft-export` (FilmCraft's encoders). Frontends (desktop, CLI, MCP, web) start here. +//! the media importer, the expression engine, JavaScript scripting (`effectcraft-script`) and +//! Render Queue export through `effectcraft-export` (FilmCraft's encoders). Frontends (desktop, +//! CLI, MCP, web) start here. use std::sync::Arc; use effectcraft_engine::project::render_queue::OutputFormat; use effectcraft_engine::{ExportJob, ExportResult, Exporter, Importer, Session}; use effectcraft_project::Footage; +pub use effectcraft_script as script; struct MediaImporter; @@ -47,7 +49,7 @@ impl Exporter for FileExporter { } } -/// A new session with media, import, expressions and export enabled. +/// A new session with media, import, expressions, scripting and export enabled. pub fn session() -> Session { Session { exporter: Some(Arc::new(FileExporter::default())), @@ -55,6 +57,7 @@ pub fn session() -> Session { importer: Some(Arc::new(MediaImporter)), expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_script::runner), ..Default::default() } } @@ -350,6 +353,64 @@ mod tests { assert!(std::path::Path::new(&px.footage.path).exists(), "{}", px.footage.path); } + /// A script builds a comp, queues it and renders it through the Render Queue. + #[test] + fn script_renders_through_the_render_queue() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-script"); + let _ = std::fs::remove_dir_all(&dir); + let out = dir.join("scripted.gif"); + let mut s = super::session(); + let code = format!( + r#" + var comp = app.project.items.addComp("Scripted", 64, 48, 1, 0.2, 10); + var sq = comp.layers.addSolid([1, 0.5, 0], "Square", 16, 16, 1); + sq.transform.position.setValueAtTime(0, [8, 24]); + sq.transform.position.setValueAtTime(0.1, [56, 24]); + var item = app.project.renderQueue.items.add(comp); + item.outputModule(1).applyTemplate("GIF"); + item.outputModule(1).file = new File({}); + app.project.renderQueue.render(); + [item.status == RQItemStatus.DONE, item.outputModule(1).file.fsName, app.project.renderQueue.numItems] + "#, + serde_json::to_string(&out.to_string_lossy()).unwrap() + ); + let r = s.execute("script.run", json!({"code": code})).unwrap(); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"][0], json!(true), "{r}"); + assert_eq!(r["result"][2], json!(1)); + let bytes = std::fs::read(&out).unwrap(); + assert!(bytes.starts_with(b"GIF89a")); + } + + /// importFile, layers.add(footage) and project save/open from a script. + #[test] + fn script_imports_footage_and_saves() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-script-import"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let q = |n: &str| serde_json::to_string(&dir.join(n).to_string_lossy()).unwrap(); + let mut s = super::session(); + let code = format!( + r#" + var c = app.project.items.addComp("Src", 32, 32, 1, 1, 10); + c.layers.addSolid([0, 1, 0], "G", 32, 32, 1); + app.run("comp.saveFrameAs", {{comp: c.id, path: {png}}}); + var still = app.project.importFile(new ImportOptions(new File({png}))); + var main = app.project.items.addComp("Main", 64, 64, 1, 1, 10); + var l = main.layers.add(still); + app.project.save(new File({proj})); + [still instanceof FootageItem, still.width, still.mainSource.isStill, l.source.name, app.project.file.name] + "#, + png = q("frame.png"), + proj = q("p.ecproj") + ); + let r = s.execute("script.run", json!({"code": code})).unwrap(); + assert_eq!(r["ok"], json!(true), "{r}"); + assert_eq!(r["result"], json!([true, 32, true, "frame.png", "p.ecproj"]), "{r}"); + let r = s.execute("script.run", json!({"code": format!("app.open(new File({})); app.project.numItems", q("p.ecproj"))})).unwrap(); + assert_eq!(r["result"], json!(5), "{r}"); // Src, Solids, G, frame.png, Main + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs new file mode 100644 index 000000000..590112c9f --- /dev/null +++ b/crates/host/tests/vector_import.rs @@ -0,0 +1,186 @@ +//! Photoshop and SVG import through the fully wired session (media decoding included): pixels +//! of imported compositions, SVG footage at any scale, and Create Shapes from Vector Layer. + +use effectcraft_engine::render::RenderOpts; +use effectcraft_project::ItemId; +use effectcraft_psd::Rect; +use effectcraft_psd::write::*; +use effectcraft_raster::Image; +use serde_json::json; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-host-vector-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +fn mean_diff(a: &Image, b: &Image) -> f32 { + assert_eq!((a.width, a.height), (b.width, b.height)); + let mut s = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + for c in 0..4 { + s += (p[c] - q[c]).abs(); + } + } + s / (a.data.len() * 4) as f32 +} + +fn psd_doc() -> WDoc { + let mut d = WDoc::new(64, 48); + let mut masked = WLayer::solid("Masked", Rect::new(30, 8, 24, 24), [0.1, 0.9, 0.3, 1.0]); + masked.mask = Some(WMask { + rect: Rect::new(30, 8, 24, 24), + data: (0..576).map(|i| if (i / 24) < 12 { 1.0 } else { 0.25 }).collect(), + default_color: 0, + disabled: false, + }); + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 64, 48), [0.2, 0.25, 0.3, 1.0]), + WLayer::solid("Half Red", Rect::new(4, 4, 30, 20), [1.0, 0.0, 0.0, 1.0]).with_opacity(128), + masked, + WLayer::group_end(), + WLayer::solid("In Group", Rect::new(10, 30, 20, 12), [0.9, 0.9, 0.1, 1.0]), + WLayer::group("Group", true, *b"norm"), + ]; + d +} + +/// Straight-alpha over of the layers as Photoshop composites them (normal blending), as the +/// merged image to compare the imported composition with. +fn reference(bytes: &[u8]) -> Vec<[f32; 4]> { + let p = effectcraft_psd::Psd::parse(bytes.to_vec()).unwrap(); + let mut acc = vec![[0.0f32; 4]; (p.width * p.height) as usize]; + for l in p.layers.iter().filter(|l| l.has_pixels() && !l.hidden) { + let px = p.layer_pixels(l.index, true).unwrap(); + let o = l.opacity as f32 / 255.0; + for (d, s) in acc.iter_mut().zip(&px.data) { + let a = s[3] * o; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (s[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +#[test] +fn psd_composition_renders_like_the_document() { + let mut doc = psd_doc(); + let bytes = write(&doc); + let merged = reference(&bytes); + doc.composite = Some(merged.clone()); + let bytes = write(&doc); + let path = tmp("layers.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + // Footage (merged image). + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + let merged_item = r["items"][0].as_u64().unwrap(); + // Composition and Composition – Retain Layer Sizes. + let mut renders = vec![]; + for kind in ["composition", "compositionLayerSizes"] { + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": kind})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + renders.push(s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default())); + } + // The merged footage in a comp of the same size. + s.execute("comp.new", json!({"name": "Merged", "width": 64, "height": 48, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": merged_item})).unwrap(); + let mcid = s.active_comp_id().unwrap(); + let merged_img = s.render(mcid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + assert!(mean_diff(&merged_img, &expected) < 2.0 / 255.0, "merged footage"); + for (i, img) in renders.iter().enumerate() { + let d = mean_diff(img, &expected); + assert!(d < 2.0 / 255.0, "import kind {i}: mean diff {d}"); + } + // A single layer as footage (Choose Layer). + let r = s.execute_checked("file.import", json!({"paths": [path], "layer": "Masked"})).unwrap(); + let it = s.project.item(ItemId(r["items"][0].as_u64().unwrap())).unwrap(); + assert_eq!(it.name, "Masked/layers.psd"); +} + +#[test] +fn psd_16bit_cmyk_composition_imports() { + let mut doc = psd_doc(); + doc.depth = 16; + doc.mode = effectcraft_psd::ColorMode::Cmyk; + doc.rle = false; + let bytes = write(&doc); + let merged = reference(&bytes); + let path = tmp("layers16.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + let d = mean_diff(&img, &expected); + assert!(d < 2.0 / 255.0, "mean diff {d}"); +} + +const SVG: &[u8] = include_bytes!("../../svg/tests/fixtures/basic-shapes.svg"); +const SVG2: &[u8] = include_bytes!("../../svg/tests/fixtures/paths-gradients.svg"); + +#[test] +fn svg_footage_and_shapes_from_vector_layer() { + for (name, bytes) in [("basic-shapes.svg", SVG), ("paths-gradients.svg", SVG2)] { + let path = tmp(name); + std::fs::write(&path, bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + let doc = effectcraft_svg::parse(bytes).unwrap(); + let (w, h) = doc.pixel_size(); + s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let foot = s.render(cid, t, RenderOpts::default()); + let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0); + assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised SVG"); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(!c.layers[1].switches.video); + let shapes = s.render(cid, t, RenderOpts::default()); + let d = mean_diff(&shapes, &direct); + assert!(d < 0.02, "{name}: shapes vs SVG mean diff {d}"); + // Scaled up 3×, the shape layer stays sharp and matches the SVG rasterised at 3×. + s.execute("prop.set", json!({"layer": sid, "path": "transform/scale", "value": [300, 300]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/position", "value": [0, 0]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/anchor", "value": [0, 0]})).unwrap(); + let big = s.render(cid, t, RenderOpts::default()); + let direct3 = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let d3 = mean_diff(&big, &direct3); + assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); + } +} + +#[test] +fn svg_footage_continuously_rasterizes() { + let path = tmp("cr.svg"); + std::fs::write(&path, SVG).unwrap(); + let doc = effectcraft_svg::parse(SVG).unwrap(); + let (w, h) = doc.pixel_size(); + let mut s = effectcraft_host::session(); + let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); + assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); +} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index adc3edcef..10998031f 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,6 +14,8 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index fa5df2382..1edcf5fd4 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -284,6 +284,22 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. +pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px[y * wu + x]; + *o = op.apply([s[0], s[1], s[2]], s[3].clamp(0.0, 1.0)); + } + }); + out +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs new file mode 100644 index 000000000..204312a58 --- /dev/null +++ b/crates/media/src/layered.rs @@ -0,0 +1,62 @@ +//! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one +//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised). + +use effectcraft_project::{AlphaMode, Footage}; +use effectcraft_raster::Image; + +use crate::convert::{AlphaOp, straight_to_image}; +use crate::{MediaError, Result}; + +fn is_svg(path: &str, bytes: &[u8]) -> bool { + path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) +} + +/// Footage for a Photoshop or SVG file (`None` for other formats). +pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let mut f = crate::probe::still_footage(path, psd.width, psd.height, image::ImageFormat::Png, true); + f.codec = if psd.psb { "PSB".into() } else { "PSD".into() }; + f.alpha = if psd.merged_alpha || !psd.layers.is_empty() { AlphaMode::Straight } else { AlphaMode::Ignore }; + return Ok(Some(f)); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true); + f.codec = "SVG".into(); + f.alpha = AlphaMode::Straight; + return Ok(Some(f)); + } + Ok(None) +} + +/// Decode a Photoshop or SVG still (`None` for other formats). +pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let px = match &footage.layer { + Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size), + None => psd.composite(), + } + .map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + // The rasteriser produces premultiplied pixels already. + let _ = op; + return Ok(Some(img)); + } + Ok(None) +} + +/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize). +pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option { + let doc = effectcraft_svg::parse(bytes).ok()?; + let (w, h) = doc.pixel_size(); + let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384)); + Some(effectcraft_svg::rasterize(&doc, sw, sh, scale)) +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 862fbfc20..6c67f50f5 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -10,10 +10,12 @@ //! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD, //! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency, //! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate -//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). +//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR); Photoshop documents (merged image or one layer) come +//! from `effectcraft-psd` and SVG from `effectcraft-svg` (rasterised at any scale). mod convert; pub mod exr_channels; +mod layered; mod pool; mod probe; @@ -45,7 +47,7 @@ impl From for MediaError { } /// File extensions recognised as stills. -pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"]; +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 478ddc094..9bdefcb69 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -386,6 +386,11 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } + // A layer of a layered still is keyed by its layer index and size mode. + FootageKind::Still if footage.layer.is_some() => { + let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64); + Loc { key: key(&footage.path, l), media_t: None } + } FootageKind::Still | FootageKind::Sequence | FootageKind::Model | FootageKind::Data => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; @@ -464,6 +469,9 @@ impl Inner { match loc.media_t { None => { let bytes = self.read(path)?; + if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? { + return Ok(img); + } let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; Ok(dynamic_to_image(&img, op)) } @@ -503,6 +511,14 @@ impl FootageSource for MediaPool { } } + fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { + if footage.missing || !effectcraft_render::is_vector_footage(footage) { + return None; + } + let bytes = self.inner.read(&footage.path).ok()?; + crate::layered::rasterize_svg(&bytes, scale).map(Arc::new) + } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { if footage.missing || !footage.has_video { return None; diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 9f1c13c3b..dff77eb64 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -70,6 +70,9 @@ pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option) -> Result { + if let Some(f) = crate::layered::probe(path, &bytes)? { + return Ok(f); + } if crate::MODEL_EXTENSIONS.contains(&ext_of(Path::new(path)).as_str()) { return probe_model(path, &bytes, &|_| None); } @@ -129,7 +132,7 @@ fn profile_of(c: &filmcraft_color::ColorInfo) -> Option Footage { +pub(crate) fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage { let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase(); // OpenEXR stores premultiplied colour by convention; other formats straight alpha. let alpha = match (has_alpha, fmt) { diff --git a/crates/opusenc/Cargo.toml b/crates/opusenc/Cargo.toml new file mode 100644 index 000000000..90b0fd026 --- /dev/null +++ b/crates/opusenc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-opusenc" +description = "Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only, fullband 48 kHz, 20 ms; RFC 7845 OpusHead) for WebM export audio" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/opusenc/README.md b/crates/opusenc/README.md new file mode 100644 index 000000000..0c76d49f4 --- /dev/null +++ b/crates/opusenc/README.md @@ -0,0 +1,33 @@ +# effectcraft-opusenc + +Clean-room, pure-Rust Opus encoder for EffectCraft's WebM export audio (layer L0, `std` only, +no `unsafe`). + +- **CELT-only** packets (TOC configuration 31: fullband 20 kHz, 20 ms, one frame per packet), + 48 kHz input, mono or stereo (stereo coded as dual L/R stereo), constant packet size from the + requested bitrate; all-zero frames become 3-byte silence packets. +- Forward MDCT with the low-overlap window, band energies with inter/intra coarse prediction + (cheaper of the two per frame), fine and final energy bits, the decoder's exact bit allocation, + spreading and recursive band splitting, greedy PVQ search and codeword indexing. +- Not implemented: SILK/hybrid modes, transient (short-block) frames, TF changes, pitch + post-filter, intensity stereo, dynamic allocation boosts, VBR rate control. +- `OpusEncoder::pre_skip()` is 120 samples (the MDCT overlap); `opus_head()` returns the RFC 7845 + identification header. + +```rust +let mut enc = effectcraft_opusenc::OpusEncoder::new(2, 128_000); +let packet = enc.encode_float(&vec![0.0; effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2]); +``` + +## Specifications + +- RFC 6716, *Definition of the Opus Audio Codec* (September 2012), with the RFC 8251 updates + (October 2017) as implemented by the decoder we round-trip against. +- RFC 7845, *Ogg Encapsulation for the Opus Audio Codec* (April 2016), §5.1 `OpusHead` and + §4.2 pre-skip. + +Implemented from the RFC text. Constant tables (band edges, `eMeans`, the coarse-energy Laplace +model, the allocation table, ICDFs) and the decoder-side allocation arithmetic are mirrored from +FilmCraft's first-party, spec-derived `filmcraft-opus` decoder (MIT OR Apache-2.0), which is also +the round-trip oracle in `crates/export/tests/opus_roundtrip.rs` (ffmpeg is a second, external +oracle when installed). No third-party encoder or decoder source was consulted. diff --git a/crates/opusenc/src/bands.rs b/crates/opusenc/src/bands.rs new file mode 100644 index 000000000..744de3fe3 --- /dev/null +++ b/crates/opusenc/src/bands.rs @@ -0,0 +1,497 @@ +//! Band shape coding: PVQ search and codeword indexing, spreading, recursive band splitting +//! (RFC 6716 §4.3.4, encoder direction). +//! +//! The encoder codes 20 ms long-block frames only (no transients, no TF changes), and stereo +//! as dual (independent L/R) stereo, so a band is always coded by `quant_partition` with a single +//! block. Every bit-budget computation mirrors the decoder so both sides take the same +//! split/pulse decisions. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::rate::{Mode, QTHETA_OFFSET, get_pulses}; +use crate::tables::*; + +pub const SPREAD_NORMAL: usize = 2; + +#[inline] +fn frac_mul16(a: i32, b: i32) -> i32 { + (16384 + (a as i16 as i32) * (b as i16 as i32)) >> 15 +} + +fn bitexact_cos(x: i32) -> i32 { + let tmp = (4096 + x * x) >> 13; + let x2 = tmp; + let x2 = (32767 - x2) + frac_mul16(x2, -7651 + frac_mul16(x2, 8277 + frac_mul16(-626, x2))); + 1 + x2 +} + +fn bitexact_log2tan(isin: i32, icos: i32) -> i32 { + let lc = ilog(icos as u32); + let ls = ilog(isin as u32); + let icos = icos << (15 - lc); + let isin = isin << (15 - ls); + (ls - lc) * (1 << 11) + frac_mul16(isin, frac_mul16(isin, -2597) + 7932) - frac_mul16(icos, frac_mul16(icos, -2597) + 7932) +} + +fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 { + const EXP2_TABLE8: [i32; 8] = [16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048]; + let mut n2 = 2 * n - 1; + if stereo && n == 2 { + n2 -= 1; + } + let mut qb = (b + n2 * offset) / n2; + qb = qb.min(b - pulse_cap - (4 << BITRES)); + qb = qb.min(8 << BITRES); + if qb < (1 << BITRES >> 1) { + 1 + } else { + let qn = EXP2_TABLE8[(qb & 0x7) as usize] >> (14 - (qb >> BITRES)); + (qn + 1) >> 1 << 1 + } +} + +/// One elementary pass of the spreading rotation, inverted (the decoder applies the forward +/// pass; the encoder undoes it before the pulse search). +fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + if len <= stride { + return; + } + if len >= 2 * stride + 1 { + for i in 0..=len - 2 * stride - 1 { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } + } + for i in (0..len - stride).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } +} + +/// Inverse of the decoder's spreading rotation (§4.3.4.3) for one block of `len` samples. +pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + // The decoder applies (stride2: s, c) then (1: c, s); undo in reverse order. + exp_rotation1_inv(x, len, 1, c, s); + if stride2 != 0 { + exp_rotation1_inv(x, len, stride2, s, c); + } +} + +/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`. +pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { + let n = x.len(); + y[..n].fill(0); + let abs: Vec = x.iter().map(|v| v.abs()).collect(); + let sum: f32 = abs.iter().sum(); + if !sum.is_finite() || sum <= 1e-15 { + y[0] = k as i32; + return; + } + let mut placed = 0usize; + let mut xy = 0f32; + let mut yy = 0f32; + if k > n / 2 { + // Pre-project onto the pyramid (rounding down) and place the rest greedily. + let r = (k as f32 - 1.0) / sum; + for j in 0..n { + let p = (abs[j] * r).floor() as i32; + y[j] = p; + placed += p as usize; + xy += abs[j] * p as f32; + yy += (p * p) as f32; + } + } + while placed < k { + let mut best = 0usize; + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } + xy += abs[best]; + yy += 2.0 * y[best] as f32 + 1.0; + y[best] += 1; + placed += 1; + } + for j in 0..n { + if x[j] < 0.0 { + y[j] = -y[j]; + } + } +} + +/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps +/// back to `y`. +pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec) { + let n = y.len(); + let w = k + 1; + v.clear(); + v.resize((n + 1) * w, 0); + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + let a = v[(nn - 1) * w + kk]; + let b = v[nn * w + kk - 1]; + let c = v[(nn - 1) * w + kk - 1]; + v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + } + } + let total = v[n * w + k]; + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut index = 0u64; + let mut kk = k; + for j in 0..n { + if kk == 0 { + break; + } + let rem = n - j; + let t = y[j].unsigned_abs() as usize; + // Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half. + let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum(); + if y[j] < 0 { + base += (vv(rem - 1, kk) + vv(rem, kk)) / 2; + } + index += base; + kk -= t; + } + debug_assert!(total <= u32::MAX as u64); + if total >= 2 { + enc.uint(index as u32, total as u32); + } +} + +pub struct BandEnc<'m, 'e> { + pub m: &'m Mode, + pub enc: &'e mut RangeEncoder, + pub i: usize, + pub spread: usize, + pub remaining_bits: i32, + pvq_v: Vec, + iy: Vec, + tmp: Vec, +} + +impl BandEnc<'_, '_> { + /// `compute_theta` for a mono split (single block): measures the energy split between the + /// two halves and codes the quantised angle with the triangular pdf. + fn compute_theta_split(&mut self, xa: &[f32], xb: &[f32], b: &mut i32, lm: i32) -> (i32, i32, i32) { + let n = xa.len() as i32; + let i = self.i; + let pulse_cap = self.m.log_n[i] + lm * (1 << BITRES); + let offset = (pulse_cap >> 1) - QTHETA_OFFSET; + let qn = compute_qn(n, *b, offset, pulse_cap, false); + let tell = self.enc.tell_frac(); + let mut itheta = 0i32; + if qn != 1 { + let mid = xa.iter().map(|v| v * v).sum::().sqrt(); + let side = xb.iter().map(|v| v * v).sum::().sqrt(); + let ang = (side as f64).atan2(mid as f64) * std::f64::consts::FRAC_2_PI; + let it = ((ang * qn as f64 + 0.5).floor() as i32).clamp(0, qn) as u32; + let q = qn as u32; + let half = q >> 1; + let ft = (half + 1) * (half + 1); + let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) }; + self.enc.encode(fl, fl + fs, ft); + itheta = ((it * 16384) / q) as i32; + } + let qalloc = self.enc.tell_frac() - tell; + *b -= qalloc; + let delta = if itheta == 0 { + -16384 + } else if itheta == 16384 { + 16384 + } else { + let imid = bitexact_cos(itheta); + let iside = bitexact_cos(16384 - itheta); + frac_mul16((n - 1) << 7, bitexact_log2tan(iside, imid)) + }; + (itheta, delta, qalloc) + } + + /// `quant_partition` (single block, encoder side). + pub fn quant_partition(&mut self, x: &[f32], mut b: i32, mut lm: i32) { + let n = x.len(); + let i = self.i; + let cache = self.m.cache(lm, i); + let cache_max = cache[cache[0] as usize] as i32; + if lm != -1 && b > cache_max + 12 && n > 2 { + let half = n >> 1; + let (xa, xb) = x.split_at(half); + lm -= 1; + let (itheta, delta, qalloc) = self.compute_theta_split(xa, xb, &mut b, lm); + let mut mbits = 0.max(b.min((b - delta) / 2)); + let mut sbits = b - mbits; + self.remaining_bits -= qalloc; + let mut rebalance = self.remaining_bits; + if mbits >= sbits { + self.quant_partition(xa, mbits, lm); + rebalance = mbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 0 { + sbits += rebalance - (3 << BITRES); + } + self.quant_partition(xb, sbits, lm); + } else { + self.quant_partition(xb, sbits, lm); + rebalance = sbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 16384 { + mbits += rebalance - (3 << BITRES); + } + self.quant_partition(xa, mbits, lm); + } + return; + } + let m = self.m; + let mut q = m.bits2pulses(i, lm, b); + let mut curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + while self.remaining_bits < 0 && q > 0 { + self.remaining_bits += curr_bits; + q -= 1; + curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + } + if q != 0 { + let k = get_pulses(q) as usize; + let mut tmp = std::mem::take(&mut self.tmp); + tmp.clear(); + tmp.extend_from_slice(x); + exp_rotation_inv(&mut tmp, n, k, self.spread); + let mut iy = std::mem::take(&mut self.iy); + iy.clear(); + iy.resize(n, 0); + pvq_search(&tmp, k, &mut iy); + encode_pulses(&iy, k, self.enc, &mut self.pvq_v); + self.iy = iy; + self.tmp = tmp; + } + } +} + +/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels +/// of `n` normalised MDCT coefficients each. +#[allow(clippy::too_many_arguments)] +pub fn quant_all_bands( + m: &Mode, + start: usize, + end: usize, + x: &[f32], + n_total: usize, + c: usize, + pulses: &[i32; NB_EBANDS], + spread: usize, + dual_stereo: bool, + intensity: usize, + total_bits: i32, + mut balance: i32, + enc: &mut RangeEncoder, + lm: usize, + coded_bands: usize, +) { + let mm = 1usize << lm; + debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo"); + let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() }; + for i in start..end { + ctx.i = i; + let xo = mm * EBANDS[i] as usize; + let n = mm * EBANDS[i + 1] as usize - xo; + let tell = ctx.enc.tell_frac(); + if i != start { + balance -= tell; + } + let remaining_bits = total_bits - tell - 1; + ctx.remaining_bits = remaining_bits; + let b = if i < coded_bands { + let curr_balance = balance / 3.min(coded_bands as i32 - i as i32); + 0.max(16383.min((remaining_bits + 1).min(pulses[i] + curr_balance))) + } else { + 0 + }; + // Bands at and above `intensity` (= coded_bands) carry no bits, so they emit no symbols. + if i < coded_bands || c == 1 { + if c == 2 && dual_stereo && i < intensity { + ctx.quant_partition(&x[xo..xo + n], b / 2, lm as i32); + ctx.quant_partition(&x[n_total + xo..n_total + xo + n], b / 2, lm as i32); + } else { + ctx.quant_partition(&x[xo..xo + n], b, lm as i32); + } + } + balance += pulses[i] + tell; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + fn exp_rotation1(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + let ms = -s; + if len <= stride { + return; + } + for i in 0..len - stride { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + if len >= 2 * stride + 1 { + for i in (0..=len - 2 * stride - 1).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + } + } + + /// The decoder's spreading rotation (single block). + fn exp_rotation(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + if stride2 != 0 { + exp_rotation1(x, len, stride2, s, c); + } + exp_rotation1(x, len, 1, c, s); + } + + #[test] + fn spreading_inverse_is_exact() { + for &len in &[8usize, 16, 24, 48, 96, 176] { + for k in [1usize, 2, 3, 5] { + let orig: Vec = (0..len).map(|i| ((i * 31 % 17) as f32 - 8.0) / 8.0).collect(); + let mut x = orig.clone(); + exp_rotation_inv(&mut x, len, k, SPREAD_NORMAL); + exp_rotation(&mut x, len, k, SPREAD_NORMAL); + for (a, b) in x.iter().zip(&orig) { + assert!((a - b).abs() < 1e-4, "len={len} k={k}"); + } + } + } + } + + /// PVQ decoder of RFC 6716 §4.3.4.2 (as in FilmCraft's decoder). + fn decode_pulses(n: usize, k: usize, dec: &mut RangeDecoder) -> Vec { + let w = k + 1; + let mut v = vec![0u64; (n + 1) * w]; + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + v[nn * w + kk] = v[(nn - 1) * w + kk].saturating_add(v[nn * w + kk - 1]).saturating_add(v[(nn - 1) * w + kk - 1]); + } + } + let total = v[n * w + k]; + let mut i = if total >= 2 { dec.uint(total as u32) as u64 } else { 0 }; + let mut y = vec![0i32; n]; + let mut kk = k; + for j in 0..n { + let rem = n - j; + if kk == 0 { + break; + } + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut p = (vv(rem - 1, kk) + vv(rem, kk)) / 2; + let neg = i >= p; + if neg { + i -= p; + } + let k0 = kk; + p -= vv(rem - 1, kk); + while p > i && kk > 0 { + kk -= 1; + p -= vv(rem - 1, kk); + } + let mag = (k0 - kk) as i32; + y[j] = if neg { -mag } else { mag }; + i -= p.min(i); + } + y + } + + fn all_codewords(n: usize, k: usize) -> Vec> { + if n == 0 { + return if k == 0 { vec![vec![]] } else { vec![] }; + } + let mut out = Vec::new(); + for t in -(k as i32)..=(k as i32) { + for mut rest in all_codewords(n - 1, k - t.unsigned_abs() as usize) { + rest.insert(0, t); + out.push(rest); + } + } + out + } + + #[test] + fn pvq_index_roundtrip_exhaustive() { + let mut v = Vec::new(); + for n in 1..6usize { + for k in 1..6usize { + for y in all_codewords(n, k) { + let mut e = RangeEncoder::new(); + encode_pulses(&y, k, &mut e, &mut v); + let buf = e.done(16); + let mut d = RangeDecoder::new(&buf); + assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}"); + } + } + } + } + + #[test] + fn pvq_search_finds_good_direction() { + let x: Vec = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect(); + let mut y = vec![0i32; 32]; + for k in [1usize, 4, 16, 64] { + pvq_search(&x, k, &mut y); + assert_eq!(y.iter().map(|v| v.unsigned_abs() as usize).sum::(), k); + let xy: f32 = x.iter().zip(&y).map(|(a, &b)| a * b as f32).sum(); + let yy: f32 = y.iter().map(|&b| (b * b) as f32).sum(); + let xx: f32 = x.iter().map(|a| a * a).sum(); + let corr = xy / (xx * yy).sqrt(); + assert!(corr > if k >= 16 { 0.9 } else { 0.2 }, "k={k} corr={corr}"); + } + } +} diff --git a/crates/opusenc/src/energy.rs b/crates/opusenc/src/energy.rs new file mode 100644 index 000000000..0057ae33a --- /dev/null +++ b/crates/opusenc/src/energy.rs @@ -0,0 +1,230 @@ +//! Band energy quantisation: Laplace-coded coarse energy with inter/intra prediction, fine +//! energy and the final refinement bits (RFC 6716 §4.3.2, encoder direction). +//! +//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction +//! state never drifts between encoder and decoder. + +use crate::range::RangeEncoder; +use crate::rate::MAX_FINE_BITS; +use crate::tables::*; + +const LAPLACE_MINP: u32 = 1; +const LAPLACE_NMIN: u32 = 16; + +fn laplace_get_freq1(fs0: u32, decay: u32) -> u32 { + let ft = 32768 - LAPLACE_MINP * (2 * LAPLACE_NMIN) - fs0; + (ft * (16384 - decay)) >> 15 +} + +/// Encodes a Laplace-distributed integer (`fs` = probability of 0 in Q15, `decay` in Q14). +/// Returns the value actually coded (very large magnitudes are clamped to the representable +/// range). +pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u32) -> i32 { + let mut fl = 0u32; + let mut coded = value; + if value != 0 { + let s: i32 = if value < 0 { -1 } else { 0 }; + let val = value.abs(); + fl = fs; + fs = laplace_get_freq1(fs, decay); + let mut i = 1; + while fs > 0 && i < val { + fs *= 2; + fl += fs + 2 * LAPLACE_MINP; + fs = (fs * decay) >> 15; + i += 1; + } + if fs == 0 { + let ndi_max = (32768 - fl + LAPLACE_MINP - 1) as i32; + let ndi_max = (ndi_max - s) >> 1; + let di = (val - i).min(ndi_max - 1); + fl = (fl as i32 + (2 * di + 1 + s) * LAPLACE_MINP as i32) as u32; + fs = LAPLACE_MINP.min(32768 - fl); + let mag = i + di; + coded = if s < 0 { -mag } else { mag }; + } else { + fs += LAPLACE_MINP; + if s == 0 { + fl += fs; + } + } + } + enc.encode_bin(fl, fl + fs, 15); + coded +} + +/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative +/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in +/// bits. +#[allow(clippy::too_many_arguments)] +pub fn quant_coarse_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + intra: bool, + enc: &mut RangeEncoder, + c: usize, + lm: usize, + budget: i32, +) { + let prob = &E_PROB_MODEL[lm][intra as usize]; + let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) }; + let mut prev = [0f32; 2]; + for i in start..end { + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let oldc = old_e[idx].max(-9.0); + let x = e[idx]; + let f = x - coef * oldc - prev[ch]; + let mut qi = (f + 0.5).floor().clamp(-64.0, 64.0) as i32; + let tell = enc.tell(); + // Keep a few bits per remaining band in reserve when the budget runs out. + let bits_left = budget - tell - 3 * c as i32 * (end - i) as i32; + if i != start && bits_left < 30 { + if bits_left < 24 { + qi = qi.min(1); + } + if bits_left < 16 { + qi = qi.max(-1); + } + } + if budget - tell >= 15 { + let pi = 2 * i.min(20); + qi = laplace_encode(enc, qi, (prob[pi] as u32) << 7, (prob[pi + 1] as u32) << 6); + } else if budget - tell >= 2 { + qi = qi.clamp(-1, 1); + enc.icdf(if qi < 0 { (-2 * qi - 1) as usize } else { (2 * qi) as usize }, &SMALL_ENERGY_ICDF, 2); + } else if budget - tell >= 1 { + qi = qi.clamp(-1, 0); + enc.bit_logp(qi == -1, 1); + } else { + qi = -1; + } + let q = qi as f32; + old_e[idx] = oldc; + let tmp = coef * old_e[idx] + prev[ch] + q; + old_e[idx] = tmp; + prev[ch] = prev[ch] + q - beta * q; + } + } +} + +/// `quant_fine_energy`: raw bits refining each band's quantised energy. +pub fn quant_fine_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + enc: &mut RangeEncoder, + c: usize, +) { + for i in start..end { + let fq = fine_quant[i]; + if fq <= 0 { + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let err = e[idx] - old_e[idx]; + let levels = 1i32 << fq; + let q2 = (((err + 0.5) * levels as f32).floor() as i32).clamp(0, levels - 1); + enc.bits(q2 as u32, fq as u32); + let offset = (q2 as f32 + 0.5) * (1 << (14 - fq)) as f32 / 16384.0 - 0.5; + old_e[idx] += offset; + } + } +} + +/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of +/// energy resolution per band, in priority order. +#[allow(clippy::too_many_arguments)] +pub fn quant_energy_finalise( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + fine_priority: &[i32; NB_EBANDS], + mut bits_left: i32, + enc: &mut RangeEncoder, + c: usize, +) { + for prio in 0..2 { + let mut i = start; + while i < end && bits_left >= c as i32 { + if fine_quant[i] >= MAX_FINE_BITS || fine_priority[i] != prio { + i += 1; + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let q2 = (e[idx] - old_e[idx] >= 0.0) as u32; + enc.bits(q2, 1); + let offset = (q2 as f32 - 0.5) * (1 << (14 - fine_quant[i] - 1)) as f32 / 16384.0; + old_e[idx] += offset; + bits_left -= 1; + } + i += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + /// Laplace decoder of RFC 6716 §4.3.2.1 (as in FilmCraft's decoder). + fn laplace_decode(dec: &mut RangeDecoder, mut fs: u32, decay: u32) -> i32 { + let mut val = 0i32; + let fm = dec.decode(32768); + let mut fl = 0u32; + if fm >= fs { + val += 1; + fl = fs; + fs = laplace_get_freq1(fs, decay) + LAPLACE_MINP; + while fs > LAPLACE_MINP && fm >= fl + 2 * fs { + fs *= 2; + fl += fs; + fs = (((fs - 2 * LAPLACE_MINP) * decay) >> 15) + LAPLACE_MINP; + val += 1; + } + if fs <= LAPLACE_MINP { + let di = (fm - fl) >> 1; + val += di as i32; + fl += 2 * di * LAPLACE_MINP; + } + if fm < fl + fs { + val = -val; + } else { + fl += fs; + } + } + dec.update(fl, (fl + fs).min(32768), 32768); + val + } + + #[test] + fn laplace_roundtrip_all_models() { + for lm in 0..4 { + for intra in 0..2 { + for band in 0..21 { + let prob = &E_PROB_MODEL[lm][intra]; + let (fs, decay) = ((prob[2 * band] as u32) << 7, (prob[2 * band + 1] as u32) << 6); + let vals: Vec = (-64..=64).collect(); + let mut e = RangeEncoder::new(); + let coded: Vec = vals.iter().map(|&v| laplace_encode(&mut e, v, fs, decay)).collect(); + let size = (e.tell() as usize).div_ceil(8); + let buf = e.done(size); + let mut d = RangeDecoder::new(&buf); + for (&v, &c) in vals.iter().zip(&coded) { + assert!(c == v || (c.signum() == v.signum() && c.abs() < v.abs()), "clamp {v} -> {c}"); + assert_eq!(laplace_decode(&mut d, fs, decay), c, "lm={lm} intra={intra} band={band} v={v}"); + } + } + } + } + } +} diff --git a/crates/opusenc/src/lib.rs b/crates/opusenc/src/lib.rs new file mode 100644 index 000000000..1e7a2cec8 --- /dev/null +++ b/crates/opusenc/src/lib.rs @@ -0,0 +1,311 @@ +//! Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only mode; RFC 7845 `OpusHead`). +//! +//! Produces fullband (20 kHz) 48 kHz Opus packets of 20 ms, mono or stereo, at a constant +//! packet size derived from the requested bitrate. Each packet is a single CELT frame (TOC +//! configuration 31, frame-count code 0). Stereo is coded as dual (independent L/R) stereo. +//! +//! Layer L0: no dependencies beyond `std`; no `unsafe`; builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use effectcraft_opusenc::OpusEncoder; +//! let mut enc = OpusEncoder::new(2, 128_000); +//! let pcm = vec![0.0f32; OpusEncoder::FRAME_SIZE * 2]; +//! let packet = enc.encode_float(&pcm); +//! assert_eq!(packet[0] >> 3, 31); // CELT-only, fullband, 20 ms +//! assert_eq!(enc.opus_head().len(), 19); +//! ``` + +// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions. +#![allow(clippy::precedence, clippy::int_plus_one)] + +mod bands; +mod energy; +mod mdct; +mod range; +mod rate; +mod tables; + +use std::sync::OnceLock; + +use bands::{SPREAD_NORMAL, quant_all_bands}; +use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy}; +use mdct::Mdct; +use range::{BITRES, RangeEncoder}; +use rate::{Mode, compute_allocation}; +use tables::*; + +/// Samples per channel in one 20 ms frame at 48 kHz. +const N: usize = 960; +/// LM of a 20 ms frame (8 short blocks). +const LM: usize = 3; +/// Smallest CELT payload the encoder produces (bytes). +const MIN_FRAME_BYTES: usize = 16; +/// Largest CELT payload allowed in one frame (RFC 6716 §3.4, R2). +const MAX_FRAME_BYTES: usize = 1275; + +fn mode() -> &'static Mode { + static MODE: OnceLock = OnceLock::new(); + MODE.get_or_init(Mode::new) +} + +/// Opus encoder for one mono or stereo stream at 48 kHz. +pub struct OpusEncoder { + channels: usize, + /// CELT payload bytes per packet (the packet is one byte longer: the TOC). + frame_bytes: usize, + mdct: Mdct, + /// Last input sample of each channel (pre-emphasis filter memory, ×32768). + preemph_mem: [f32; 2], + /// Last `OVERLAP` pre-emphasised samples of each channel. + hist: Vec, + /// Quantised band energies, exactly as the decoder holds them. + old_band_e: [f32; 2 * NB_EBANDS], + block: Vec, + freq: Vec, +} + +impl OpusEncoder { + /// Samples per channel per packet (20 ms at 48 kHz). + pub const FRAME_SIZE: usize = N; + + /// Creates an encoder for `channels` (1 or 2) channels of 48 kHz audio at `bitrate_bps` + /// (total for all channels; packets are constant-size, clamped to 6.8–510 kb/s). + pub fn new(channels: u8, bitrate_bps: u32) -> Self { + assert!(channels == 1 || channels == 2, "Opus family-0 streams have 1 or 2 channels"); + let c = channels as usize; + let packet_bytes = (bitrate_bps as usize).div_ceil(400); + let frame_bytes = packet_bytes.saturating_sub(1).clamp(MIN_FRAME_BYTES, MAX_FRAME_BYTES); + OpusEncoder { + channels: c, + frame_bytes, + mdct: Mdct::new(N), + preemph_mem: [0.0; 2], + hist: vec![0.0; c * OVERLAP], + old_band_e: [-28.0; 2 * NB_EBANDS], + block: vec![0.0; N + OVERLAP], + freq: vec![0.0; c * N], + } + } + + /// Number of channels. + pub fn channels(&self) -> u8 { + self.channels as u8 + } + + /// Actual bitrate in bits per second (packet size × 50 packets per second). + pub fn bitrate(&self) -> u32 { + ((self.frame_bytes + 1) * 8 * 50) as u32 + } + + /// Samples at 48 kHz the decoder must discard from the start of the stream (the MDCT + /// overlap); this is the `OpusHead` pre-skip. + pub fn pre_skip(&self) -> u16 { + OVERLAP as u16 + } + + /// The 19-byte `OpusHead` identification header (RFC 7845 §5.1): version 1, channel count, + /// pre-skip, input sample rate 48000, output gain 0, channel mapping family 0. + pub fn opus_head(&self) -> Vec { + let mut h = Vec::with_capacity(19); + h.extend_from_slice(b"OpusHead"); + h.push(1); + h.push(self.channels as u8); + h.extend_from_slice(&self.pre_skip().to_le_bytes()); + h.extend_from_slice(&48_000u32.to_le_bytes()); + h.extend_from_slice(&0i16.to_le_bytes()); + h.push(0); + h + } + + /// Encodes one 20 ms frame of interleaved f32 samples in [-1, 1] (exactly + /// `FRAME_SIZE * channels` values) and returns one Opus packet (TOC byte + CELT frame). + pub fn encode_float(&mut self, pcm: &[f32]) -> Vec { + let c = self.channels; + assert_eq!(pcm.len(), N * c, "encode_float takes exactly FRAME_SIZE samples per channel"); + + // Pre-emphasis, the MDCT and the band energies. + let mut band_e = [0f32; 2 * NB_EBANDS]; + let mut log_e = [0f32; 2 * NB_EBANDS]; + let mut peak = 0f32; + for ch in 0..c { + self.block[..OVERLAP].copy_from_slice(&self.hist[ch * OVERLAP..(ch + 1) * OVERLAP]); + let mut mem = self.preemph_mem[ch]; + for j in 0..N { + let s = pcm[j * c + ch]; + let s = if s.is_finite() { s.clamp(-2.0, 2.0) * 32768.0 } else { 0.0 }; + let v = s - PREEMPH * mem; + mem = s; + self.block[OVERLAP + j] = v; + } + self.preemph_mem[ch] = mem; + self.hist[ch * OVERLAP..(ch + 1) * OVERLAP].copy_from_slice(&self.block[N..]); + peak = self.block.iter().fold(peak, |p, v| p.max(v.abs())); + let freq = &mut self.freq[ch * N..(ch + 1) * N]; + self.mdct.forward(&self.block, freq); + for i in 0..NB_EBANDS { + let lo = (EBANDS[i] as usize) << LM; + let hi = (EBANDS[i + 1] as usize) << LM; + let e = (1e-27f32 + freq[lo..hi].iter().map(|v| v * v).sum::()).sqrt(); + band_e[ch * NB_EBANDS + i] = e; + log_e[ch * NB_EBANDS + i] = e.log2() - E_MEANS[i]; + let g = 1.0 / e; + for v in freq[lo..hi].iter_mut() { + *v *= g; + } + } + } + let silence = peak < 1e-4; + let toc = (31u8 << 3) | if c == 2 { 4 } else { 0 }; + let mut packet = vec![toc]; + packet.extend_from_slice(&self.encode_celt(&log_e, silence)); + packet + } + + /// Codes one CELT frame (RFC 6716 §4.3, in bitstream order). + fn encode_celt(&mut self, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec { + let m = mode(); + let c = self.channels; + let (start, end) = (0usize, NB_EBANDS); + let mut enc = RangeEncoder::new(); + if c == 1 { + for i in 0..NB_EBANDS { + self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]); + } + } + if silence { + // The silence flag; the decoder treats every remaining bit as consumed. + enc.bit_logp(true, 15); + self.old_band_e = [-28.0; 2 * NB_EBANDS]; + return enc.done(2); + } + let len = self.frame_bytes; + let total_bits_raw = len as i32 * 8; + enc.bit_logp(false, 15); + // No pitch post-filter. + if enc.tell() + 16 <= total_bits_raw { + enc.bit_logp(false, 1); + } + // Long blocks only. + if enc.tell() + 3 <= total_bits_raw { + enc.bit_logp(false, 3); + } + // Coarse energy, choosing intra or inter prediction by trial (whichever is cheaper). + if enc.tell() + 3 <= total_bits_raw { + let mut enc_intra = enc.clone(); + let mut old_intra = self.old_band_e; + enc_intra.bit_logp(true, 3); + quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw); + enc.bit_logp(false, 3); + let mut old_inter = self.old_band_e; + quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw); + if enc_intra.tell_frac() < enc.tell_frac() { + enc = enc_intra; + self.old_band_e = old_intra; + } else { + self.old_band_e = old_inter; + } + } else { + quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw); + } + // TF resolution: no changes. + { + let mut budget = total_bits_raw; + let mut tell = enc.tell(); + let mut logp = 4; + let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget; + budget -= tf_select_rsv as i32; + for _ in start..end { + if tell + logp <= budget { + enc.bit_logp(false, logp as u32); + tell = enc.tell(); + } + logp = 5; + } + if tf_select_rsv && TF_SELECT_TABLE[LM][0] != TF_SELECT_TABLE[LM][2] { + enc.bit_logp(false, 1); + } + } + if enc.tell() + 4 <= total_bits_raw { + enc.icdf(SPREAD_NORMAL, &SPREAD_ICDF, 5); + } + // Dynamic allocation: no boosts. + let cap = m.init_caps(LM, c); + let offsets = [0i32; NB_EBANDS]; + let total_bits = total_bits_raw << BITRES; + let mut tellf = enc.tell_frac(); + for &cp in cap.iter().take(end).skip(start) { + if tellf + (6 << BITRES) < total_bits && 0 < cp { + enc.bit_logp(false, 6); + tellf = enc.tell_frac(); + } + } + let alloc_trim = 5; + if tellf + (6 << BITRES) <= total_bits { + enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7); + } + let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1; + let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc); + quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c); + quant_all_bands( + m, + start, + end, + &self.freq, + N, + c, + &alloc.pulses, + SPREAD_NORMAL, + alloc.dual_stereo, + alloc.intensity, + total_bits_raw << BITRES, + alloc.balance, + &mut enc, + LM, + alloc.coded_bands, + ); + let bits_left = total_bits_raw - enc.tell(); + quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c); + debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw); + if c == 1 { + let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS); + b.copy_from_slice(a); + } + enc.done(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opus_head_layout() { + let e = OpusEncoder::new(2, 96_000); + let h = e.opus_head(); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(h[8], 1); + assert_eq!(h[9], 2); + assert_eq!(u16::from_le_bytes([h[10], h[11]]), e.pre_skip()); + assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000); + assert_eq!(&h[16..], &[0, 0, 0]); + } + + #[test] + fn packet_sizes_follow_bitrate() { + for (ch, rate) in [(1u8, 64_000u32), (2, 128_000), (2, 256_000)] { + let mut e = OpusEncoder::new(ch, rate); + let pcm: Vec = (0..N * ch as usize).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); + let p = e.encode_float(&pcm); + assert_eq!(p.len(), rate as usize / 400); + assert_eq!(p[0], 0xF8 | if ch == 2 { 4 } else { 0 }); + assert_eq!(e.bitrate(), rate); + } + } + + #[test] + fn silence_is_a_tiny_packet() { + let mut e = OpusEncoder::new(1, 64_000); + let p = e.encode_float(&vec![0.0; N]); + assert_eq!(p.len(), 3); + } +} diff --git a/crates/opusenc/src/mdct.rs b/crates/opusenc/src/mdct.rs new file mode 100644 index 000000000..814de0375 --- /dev/null +++ b/crates/opusenc/src/mdct.rs @@ -0,0 +1,293 @@ +//! Forward MDCT with the CELT low-overlap window (RFC 6716 §4.3.7, run in the analysis +//! direction). +//! +//! A 20 ms frame is a 1920-point MDCT (960 coefficients) whose window is zero for 420 samples, +//! rises over the 120-sample overlap, is flat for 840 samples, falls over the next 120 samples +//! and is zero again for the last 420. The input block is therefore the 120 overlap samples kept +//! from the previous frame followed by the 960 new samples. The MDCT is folded into a DCT-IV, +//! which is computed with an N/2-point complex FFT. + +use crate::tables::OVERLAP; + +#[derive(Clone, Copy, Default, Debug)] +pub struct Cpx { + pub re: f32, + pub im: f32, +} + +impl Cpx { + #[inline] + fn mul(self, o: Cpx) -> Cpx { + Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re } + } + #[inline] + fn add(self, o: Cpx) -> Cpx { + Cpx { re: self.re + o.re, im: self.im + o.im } + } +} + +/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive). +pub struct Fft { + n: usize, + factors: Vec, + twiddle: Vec, +} + +impl Fft { + pub fn new(n: usize) -> Fft { + let mut factors = Vec::new(); + let mut m = n; + for p in [4usize, 2, 3, 5] { + while m.is_multiple_of(p) { + factors.push(p); + m /= p; + } + } + assert_eq!(m, 1, "unsupported FFT size {n}"); + let twiddle = (0..n) + .map(|k| { + let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + Fft { n, factors, twiddle } + } + + pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec) { + scratch.clear(); + scratch.extend_from_slice(&data[..self.n]); + self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1); + } + + fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) { + let p = self.factors[fi]; + let m = n / p; + if m == 1 { + for (i, o) in out.iter_mut().enumerate().take(p) { + *o = inp[i * in_stride]; + } + } else { + for q in 0..p { + self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p); + } + } + let big = self.n; + let mut t = [Cpx::default(); 5]; + for k in 0..m { + for (q, tq) in t.iter_mut().enumerate().take(p) { + *tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]); + } + for q2 in 0..p { + let mut acc = Cpx::default(); + for (q, tq) in t.iter().enumerate().take(p) { + acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)])); + } + out[q2 * m + k] = acc; + } + } + } +} + +/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples. +pub struct Mdct { + /// Number of coefficients (half the MDCT length). + n: usize, + fft: Fft, + pre: Vec, + post: Vec, + /// Rising half of the low-overlap window (RFC 6716 §4.3.7). + pub window: [f32; OVERLAP], + scale: f32, + z: Vec, + scratch: Vec, + v: Vec, +} + +impl Mdct { + pub fn new(n: usize) -> Mdct { + let n2 = n / 2; + let pi = std::f64::consts::PI; + let pre = (0..n2) + .map(|m| { + let ph = -pi * m as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let post = (0..n2) + .map(|p| { + let ph = -pi * (p as f64 + 0.25) / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let mut window = [0f32; OVERLAP]; + for (i, w) in window.iter_mut().enumerate() { + let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64; + let s = x.sin(); + *w = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32; + } + Mdct { n, fft: Fft::new(n2), pre, post, window, scale: 2.0 / n as f32, z: Vec::new(), scratch: Vec::new(), v: vec![0.0; n] } + } + + /// Transforms `block` (`n + OVERLAP` samples: the previous frame's last `OVERLAP` samples, then + /// `n` new ones) into `out[..n]`, scaled so that the RFC 6716 decoder's inverse MDCT and + /// overlap-add reproduce the input. + pub fn forward(&mut self, block: &[f32], out: &mut [f32]) { + let n = self.n; + let n2 = n / 2; + debug_assert_eq!(block.len(), n + OVERLAP); + // Place the windowed block into the (virtual) 2n-sample MDCT input u at offset `pad`. + let pad = (n - OVERLAP) / 2; + let w = &self.window; + let u = |t: usize| -> f32 { + if t < pad || t >= pad + n + OVERLAP { + return 0.0; + } + let j = t - pad; + let g = if j < OVERLAP { + w[j] + } else if j >= n { + w[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + block[j] * g + }; + // Fold to a DCT-IV input: v = (-c_r - d, a - b_r) over the four quarters of u. + for m in 0..n2 { + self.v[m] = -u(n + n2 - 1 - m) - u(n + n2 + m); + self.v[n2 + m] = u(m) - u(n2 + n2 - 1 - m); + } + // DCT-IV via an n/2-point complex FFT. + self.z.clear(); + for m in 0..n2 { + let wv = Cpx { re: self.v[2 * m], im: self.v[n - 1 - 2 * m] }; + self.z.push(wv.mul(self.pre[m])); + } + self.fft.process(&mut self.z, &mut self.scratch); + for p in 0..n2 { + let y = self.z[p].mul(self.post[p]); + out[2 * p] = y.re * self.scale; + out[n - 1 - 2 * p] = -y.im * self.scale; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Direct MDCT definition X[k] = sum u[t] cos(pi/N (t + 1/2 + N/2)(k + 1/2)). + fn direct(block: &[f32], n: usize, window: &[f32; OVERLAP]) -> Vec { + let pad = (n - OVERLAP) / 2; + let mut u = vec![0f64; 2 * n]; + for (j, &b) in block.iter().enumerate() { + let g = if j < OVERLAP { + window[j] + } else if j >= n { + window[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + u[pad + j] = (b * g) as f64; + } + (0..n) + .map(|k| { + (0..2 * n).map(|t| u[t] * (std::f64::consts::PI / n as f64 * (t as f64 + 0.5 + n as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum::() * 2.0 + / n as f64 + }) + .collect() + } + + #[test] + fn matches_direct_formula() { + for &n in &[240usize, 960] { + let block: Vec = (0..n + OVERLAP).map(|i| ((i * 37 % 101) as f32 - 50.0) / 7.0).collect(); + let mut m = Mdct::new(n); + let mut out = vec![0f32; n]; + m.forward(&block, &mut out); + let d = direct(&block, n, &m.window); + for k in 0..n { + assert!((out[k] as f64 - d[k]).abs() < 1e-2, "n={n} k={k}: {} vs {}", out[k], d[k]); + } + } + } + + // ---- Replica of the RFC 6716 decoder inverse MDCT (as in FilmCraft's decoder) ---- + + fn imdct_backward(n_full: usize, input: &[f32], out: &mut [f32], window: &[f32]) { + let n2 = n_full >> 1; + let n4 = n_full >> 2; + let t: Vec = (0..n2).map(|i| (2.0 * std::f64::consts::PI * (i as f64 + 0.125) / n_full as f64).cos() as f32).collect(); + let fft = Fft::new(n4); + let mut z = Vec::new(); + for i in 0..n4 { + let x1 = input[2 * i]; + let x2 = input[n2 - 1 - 2 * i]; + let yr = x2 * t[i] + x1 * t[n4 + i]; + let yi = x1 * t[i] - x2 * t[n4 + i]; + z.push(Cpx { re: yi, im: yr }); + } + fft.process(&mut z, &mut Vec::new()); + let overlap = OVERLAP; + let base = overlap >> 1; + let y = &mut out[base..base + n2]; + for i in 0..n4.div_ceil(2) { + let front = z[i]; + let back = z[n4 - 1 - i]; + let (re, im) = (front.im, front.re); + let (t0, t1) = (t[i], t[n4 + i]); + let yr0 = re * t0 + im * t1; + let yi0 = re * t1 - im * t0; + let (re, im) = (back.im, back.re); + let (t0, t1) = (t[n4 - i - 1], t[n2 - i - 1]); + let yr1 = re * t0 + im * t1; + let yi1 = re * t1 - im * t0; + y[2 * i] = yr0; + y[n2 - 1 - 2 * i] = yi0; + y[n2 - 2 - 2 * i] = yr1; + y[2 * i + 1] = yi1; + } + for i in 0..overlap / 2 { + let x1 = out[overlap - 1 - i]; + let x2 = out[i]; + let w1 = window[i]; + let w2 = window[overlap - 1 - i]; + out[i] = x2 * w2 - x1 * w1; + out[overlap - 1 - i] = x2 * w1 + x1 * w2; + } + } + + /// MDCT analysis followed by the decoder's IMDCT + TDAC overlap-add reproduces the input + /// delayed by exactly `OVERLAP` samples. + #[test] + fn perfect_reconstruction_through_decoder_imdct() { + let n = 960; + let frames = 5; + let sig: Vec = (0..n * frames).map(|i| (i as f32 * 0.013).sin() * 1000.0 + ((i * 7919) % 211) as f32 - 105.0).collect(); + let mut m = Mdct::new(n); + let mut hist = vec![0f32; OVERLAP]; + // Decoder memory: [frame output n][folded tail OVERLAP/2] laid out like decode_mem. + let mut mem = vec![0f32; n + OVERLAP]; + let mut decoded = Vec::new(); + for f in 0..frames { + let mut block = hist.clone(); + block.extend_from_slice(&sig[f * n..(f + 1) * n]); + hist.copy_from_slice(&block[n..]); + let mut coefs = vec![0f32; n]; + m.forward(&block, &mut coefs); + // Shift: the previous folded tail (at n..n+OVERLAP/2) moves to 0..OVERLAP/2. + mem.copy_within(n..n + OVERLAP / 2, 0); + imdct_backward(2 * n, &coefs, &mut mem, &m.window); + decoded.extend_from_slice(&mem[..n]); + } + let mut err = 0f64; + let mut sum = 0f64; + for t in OVERLAP..n * frames { + let d = decoded[t] as f64 - sig[t - OVERLAP] as f64; + err += d * d; + sum += (sig[t - OVERLAP] as f64).powi(2); + } + let snr = 10.0 * (sum / err).log10(); + assert!(snr > 80.0, "reconstruction SNR {snr:.1} dB"); + } +} diff --git a/crates/opusenc/src/range.rs b/crates/opusenc/src/range.rs new file mode 100644 index 000000000..daedd58ae --- /dev/null +++ b/crates/opusenc/src/range.rs @@ -0,0 +1,472 @@ +//! Range encoder (RFC 6716 §5.1), the exact inverse of the §4.1 range decoder. +//! +//! Range-coded symbols are written from the front of the buffer and raw bits from the back; the +//! bit accounting ([`RangeEncoder::tell`], [`RangeEncoder::tell_frac`]) matches the decoder's, so +//! every budget decision the decoder takes can be mirrored exactly. + +const SYM_BITS: u32 = 8; +const CODE_BITS: u32 = 32; +const SYM_MAX: u32 = (1 << SYM_BITS) - 1; +const CODE_SHIFT: u32 = CODE_BITS - SYM_BITS - 1; +const CODE_TOP: u32 = 1 << (CODE_BITS - 1); +const CODE_BOT: u32 = CODE_TOP >> SYM_BITS; +/// Bit resolution of [`RangeEncoder::tell_frac`] (1/8 bit). +pub const BITRES: u32 = 3; + +/// Number of bits needed to represent `x` (0 for 0). +#[inline] +pub fn ilog(x: u32) -> i32 { + (32 - x.leading_zeros()) as i32 +} + +#[derive(Clone, Debug)] +pub struct RangeEncoder { + out: Vec, + end: Vec, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + low: u32, + rem: i32, + ext: u32, +} + +impl Default for RangeEncoder { + fn default() -> Self { + Self::new() + } +} + +impl RangeEncoder { + pub fn new() -> Self { + RangeEncoder { + out: Vec::new(), + end: Vec::new(), + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1) as i32, + rng: CODE_TOP, + low: 0, + rem: -1, + ext: 0, + } + } + + /// Current range (equals the decoder's range after the same symbols). + #[cfg(test)] + pub fn rng(&self) -> u32 { + self.rng + } + + fn carry_out(&mut self, c: u32) { + if c != SYM_MAX { + let carry = c >> SYM_BITS; + if self.rem >= 0 { + self.out.push((self.rem as u32 + carry) as u8); + } + if self.ext > 0 { + let sym = (SYM_MAX + carry) & SYM_MAX; + for _ in 0..self.ext { + self.out.push(sym as u8); + } + self.ext = 0; + } + self.rem = (c & SYM_MAX) as i32; + } else { + self.ext += 1; + } + } + + #[inline] + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.carry_out(self.low >> CODE_SHIFT); + self.low = (self.low << SYM_BITS) & (CODE_TOP - 1); + self.rng <<= SYM_BITS; + self.nbits_total += SYM_BITS as i32; + } + } + + /// Encodes the symbol with cumulative range `[fl, fh)` of total `ft`. + pub fn encode(&mut self, fl: u32, fh: u32, ft: u32) { + let r = self.rng / ft; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Like [`Self::encode`] with `ft = 1 << bits`. + pub fn encode_bin(&mut self, fl: u32, fh: u32, bits: u32) { + let r = self.rng >> bits; + let ft = 1u32 << bits; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Encodes a binary symbol whose probability of being 1 is `1/2^logp`. + pub fn bit_logp(&mut self, val: bool, logp: u32) { + let r = self.rng; + let s = r >> logp; + let r2 = r - s; + if val { + self.low += r2; + self.rng = s; + } else { + self.rng = r2; + } + self.normalize(); + } + + /// Encodes symbol `s` with an "inverse CDF" table (`ft = 1 << ftb`). + pub fn icdf(&mut self, s: usize, icdf: &[u8], ftb: u32) { + let r = self.rng >> ftb; + if s > 0 { + self.low += self.rng - r * icdf[s - 1] as u32; + self.rng = r * (icdf[s - 1] as u32 - icdf[s] as u32); + } else { + self.rng -= r * icdf[s] as u32; + } + self.normalize(); + } + + /// Encodes a uniformly distributed integer `fl` in `[0, ft)` (`ft >= 2`). + pub fn uint(&mut self, fl: u32, ft: u32) { + debug_assert!(ft > 1 && fl < ft); + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let v = fl >> ftb; + self.encode(v, v + 1, top); + self.bits(fl & ((1 << ftb) - 1), ftb); + } else { + self.encode(fl, fl + 1, ft); + } + } + + /// Writes `bits` raw bits (0..=25) to the end of the buffer. + pub fn bits(&mut self, val: u32, bits: u32) { + if bits == 0 { + return; + } + let mut window = self.end_window; + let mut used = self.nend_bits; + if used as u32 + bits > CODE_BITS { + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + } + window |= val << used; + used += bits as i32; + self.end_window = window; + self.nend_bits = used; + self.nbits_total += bits as i32; + } + + /// Number of bits written so far (rounded up), identical to the decoder's `tell`. + #[inline] + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + + /// Number of 1/8 bits written so far (rounded up), identical to the decoder's `tell_frac`. + pub fn tell_frac(&self) -> i32 { + let nbits = (self.nbits_total as u32) << BITRES; + let mut l = ilog(self.rng) as u32; + let mut r = self.rng >> (l - 16); + for _ in 0..BITRES { + r = (r * r) >> 15; + let b = r >> 16; + l = (l << 1) | b; + r >>= b; + } + nbits.wrapping_sub(l) as i32 + } + + /// Finalises the stream into a buffer of exactly `size` bytes (range data at the front, raw + /// bits at the back, zero padding in between; a partial raw-bit byte is merged into the byte + /// just before the full raw-bit bytes). + pub fn done(mut self, size: usize) -> Vec { + let mut l = CODE_BITS as i32 - ilog(self.rng); + let mut msk = (CODE_TOP - 1) >> l; + let mut end = (self.low.wrapping_add(msk)) & !msk; + if (end | msk) >= self.low.wrapping_add(self.rng) { + l += 1; + msk >>= 1; + end = (self.low.wrapping_add(msk)) & !msk; + } + while l > 0 { + self.carry_out(end >> CODE_SHIFT); + end = (end << SYM_BITS) & (CODE_TOP - 1); + l -= SYM_BITS as i32; + } + if self.rem >= 0 || self.ext > 0 { + self.carry_out(0); + } + let mut window = self.end_window; + let mut used = self.nend_bits; + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + let mut buf = vec![0u8; size]; + let full = self.end.len().min(size); + let room = size - full; + let n = self.out.len().min(room); + buf[..n].copy_from_slice(&self.out[..n]); + for (k, &b) in self.end.iter().take(full).enumerate() { + buf[size - 1 - k] = b; + } + if used > 0 && room > 0 { + buf[room - 1] |= window as u8; + } + buf + } +} + +#[cfg(test)] +pub(crate) mod dec { + //! Minimal range decoder (RFC 6716 §4.1) used to verify the encoder. + use super::*; + const CODE_EXTRA: u32 = (CODE_BITS - 2) % SYM_BITS + 1; + + pub struct RangeDecoder<'a> { + buf: &'a [u8], + offs: usize, + end_offs: usize, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + val: u32, + rem: u32, + } + + impl<'a> RangeDecoder<'a> { + pub fn new(buf: &'a [u8]) -> Self { + let mut d = RangeDecoder { + buf, + offs: 0, + end_offs: 0, + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1 - ((CODE_BITS - CODE_EXTRA) / SYM_BITS) * SYM_BITS) as i32, + rng: 1 << CODE_EXTRA, + val: 0, + rem: 0, + }; + d.rem = d.read_byte(); + d.val = d.rng - 1 - (d.rem >> (SYM_BITS - CODE_EXTRA)); + d.normalize(); + d + } + fn read_byte(&mut self) -> u32 { + if self.offs < self.buf.len() { + self.offs += 1; + self.buf[self.offs - 1] as u32 + } else { + 0 + } + } + fn read_byte_from_end(&mut self) -> u32 { + if self.end_offs < self.buf.len() { + self.end_offs += 1; + self.buf[self.buf.len() - self.end_offs] as u32 + } else { + 0 + } + } + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.nbits_total += SYM_BITS as i32; + self.rng <<= SYM_BITS; + let sym = self.rem; + self.rem = self.read_byte(); + let sym = ((sym << SYM_BITS) | self.rem) >> (SYM_BITS - CODE_EXTRA); + self.val = ((self.val << SYM_BITS).wrapping_add(SYM_MAX & !sym)) & (CODE_TOP - 1); + } + } + pub fn rng(&self) -> u32 { + self.rng + } + pub fn decode(&mut self, ft: u32) -> u32 { + let ext = self.rng / ft; + let s = self.val / ext; + ft - (s + 1).min(ft) + } + pub fn update(&mut self, fl: u32, fh: u32, ft: u32) { + let ext = self.rng / ft; + let s = ext * (ft - fh); + self.val = self.val.wrapping_sub(s); + self.rng = if fl > 0 { ext * (fh - fl) } else { self.rng - s }; + self.normalize(); + } + pub fn bit_logp(&mut self, logp: u32) -> bool { + let r = self.rng; + let d = self.val; + let s = r >> logp; + let ret = d < s; + if !ret { + self.val = d - s; + } + self.rng = if ret { s } else { r - s }; + self.normalize(); + ret + } + pub fn icdf(&mut self, icdf: &[u8], ftb: u32) -> usize { + let mut s = self.rng; + let d = self.val; + let r = s >> ftb; + let mut ret = 0usize; + let mut t; + loop { + t = s; + s = r * icdf[ret] as u32; + if d >= s { + break; + } + ret += 1; + } + self.val = d - s; + self.rng = t - s; + self.normalize(); + ret + } + pub fn uint(&mut self, ft: u32) -> u32 { + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let s = self.decode(top); + self.update(s, s + 1, top); + (s << ftb) | self.bits(ftb) + } else { + let s = self.decode(ft); + self.update(s, s + 1, ft); + s + } + } + pub fn bits(&mut self, bits: u32) -> u32 { + let mut window = self.end_window; + let mut available = self.nend_bits; + if (available as u32) < bits { + loop { + window |= self.read_byte_from_end() << available; + available += SYM_BITS as i32; + if available > (CODE_BITS - SYM_BITS) as i32 { + break; + } + } + } + let ret = window & ((1u32 << bits) - 1); + self.end_window = window >> bits; + self.nend_bits = available - bits as i32; + self.nbits_total += bits as i32; + ret + } + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + } +} + +#[cfg(test)] +mod tests { + use super::dec::RangeDecoder; + use super::*; + + struct Lcg(u32); + impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(1664525).wrapping_add(1013904223); + self.0 >> 8 + } + } + + #[test] + fn starts_at_one_bit() { + let e = RangeEncoder::new(); + assert_eq!(e.tell(), 1); + assert_eq!(e.tell_frac(), 8); + } + + #[test] + fn roundtrip_mixed_symbols_and_tell_matches() { + let mut rng = Lcg(7); + let icdf: [u8; 4] = [200, 120, 30, 0]; + for trial in 0..300 { + let n = 1 + (rng.next() % 300) as usize; + let mut ops = Vec::new(); + for _ in 0..n { + match rng.next() % 5 { + 0 => { + let ft = 2 + rng.next() % 1000; + ops.push((0u8, rng.next() % ft, ft)); + } + 1 => { + let logp = 1 + rng.next() % 15; + ops.push((1, rng.next().is_multiple_of(5) as u32, logp)); + } + 2 => { + let bits = 1 + rng.next() % 24; + ops.push((2, rng.next() & ((1 << bits) - 1), bits)); + } + 3 => { + let ft = 2 + rng.next() % 100_000_000; + ops.push((3, rng.next() % ft, ft)); + } + _ => ops.push((4, rng.next() % 4, 0)), + } + } + let mut e = RangeEncoder::new(); + let mut tells = Vec::new(); + for &(k, v, p) in &ops { + match k { + 0 => e.encode(v, v + 1, p), + 1 => e.bit_logp(v != 0, p), + 2 => e.bits(v, p), + 3 => e.uint(v, p), + _ => e.icdf(v as usize, &icdf, 8), + } + tells.push(e.tell()); + } + // Exactly the number of bytes `tell` says were used must suffice. + let size = (e.tell() as usize).div_ceil(8); + let final_rng = e.rng(); + let buf = e.done(size); + assert_eq!(buf.len(), size); + let mut d = RangeDecoder::new(&buf); + for (i, &(k, v, p)) in ops.iter().enumerate() { + let got = match k { + 0 => { + let s = d.decode(p); + d.update(s, s + 1, p); + s + } + 1 => d.bit_logp(p) as u32, + 2 => d.bits(p), + 3 => d.uint(p), + _ => d.icdf(&icdf, 8) as u32, + }; + assert_eq!(got, v, "trial {trial} op {i} kind {k}"); + assert_eq!(d.tell(), tells[i], "tell mismatch trial {trial} op {i}"); + } + assert_eq!(d.rng(), final_rng); + } + } +} diff --git a/crates/opusenc/src/rate.rs b/crates/opusenc/src/rate.rs new file mode 100644 index 000000000..d8665bf13 --- /dev/null +++ b/crates/opusenc/src/rate.rs @@ -0,0 +1,561 @@ +//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1). +//! +//! The allocation must be bit-identical to the decoder's, so this mirrors FilmCraft's +//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only +//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::tables::*; + +const MAX_PSEUDO: i32 = 40; +const LOG_MAX_PSEUDO: i32 = 6; +pub const MAX_FINE_BITS: i32 = 8; +const FINE_OFFSET: i32 = 21; +pub const QTHETA_OFFSET: i32 = 4; +pub const QTHETA_OFFSET_TWOPHASE: i32 = 16; +const ALLOC_STEPS: i32 = 6; + +/// Number of pulses represented by pseudo-pulse index `i`. +pub fn get_pulses(i: i32) -> i32 { + if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) } +} + +/// Conservative (rounded up) log2 of `val` with `frac` fractional bits. +pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 { + let mut l = ilog(val); + if val & val.wrapping_sub(1) != 0 { + if l > 16 { + val = ((val - 1) >> (l - 16)) + 1; + } else { + val <<= 16 - l; + } + let mut l = (l - 1) << frac; + loop { + let b = (val >> 16) as i32; + l += b << frac; + val = (val + b as u32) >> b; + val = (val * val + 0x7FFF) >> 15; + if frac == 0 { + break; + } + frac -= 1; + } + l + (val > 0x8000) as i32 + } else { + l -= 1; + l << frac + } +} + +/// PVQ codebook size V(N, K) (saturating far above 2^32). +pub fn pvq_v(n: usize, k: usize) -> u128 { + if k == 0 { + return 1; + } + if n == 0 { + return 0; + } + let mut row = vec![0u128; k + 1]; + row[0] = 1; + for _ in 0..n { + let mut prev_diag = row[0]; + for kk in 1..=k { + let up = row[kk]; + let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag); + prev_diag = up; + row[kk] = v.min(u128::MAX >> 8); + } + row[0] = 1; + } + row[k] +} + +fn fits_in32(n: usize, k: i32) -> bool { + pvq_v(n, k as usize) <= u32::MAX as u128 +} + +/// Precomputed per-mode data. +pub struct Mode { + /// log2 of band width in 1/8 bits. + pub log_n: [i32; NB_EBANDS], + cache_index: Vec, + cache_bits: Vec, + caps: Vec, +} + +impl Mode { + pub fn new() -> Mode { + let mut log_n = [0i32; NB_EBANDS]; + for (i, l) in log_n.iter_mut().enumerate() { + *l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES); + } + let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32); + let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() }; + m.caps = compute_caps(&m); + m + } + + #[inline] + pub fn cache(&self, lm: i32, band: usize) -> &[u8] { + let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band]; + &self.cache_bits[idx.max(0) as usize..] + } + + pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 { + let cache = self.cache(lm, band); + let mut lo = 0i32; + let mut hi = cache[0] as i32; + let bits = bits - 1; + for _ in 0..LOG_MAX_PSEUDO { + let mid = (lo + hi + 1) >> 1; + if cache[mid as usize] as i32 >= bits { + hi = mid; + } else { + lo = mid; + } + } + let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 }; + if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi } + } + + pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 { + if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 } + } + + /// Per-band allocation caps for this frame (`init_caps`). + pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] { + let mut cap = [0i32; NB_EBANDS]; + for (i, cp) in cap.iter_mut().enumerate() { + let n = (EBANDS[i + 1] - EBANDS[i]) << lm; + *cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2; + } + cap + } +} + +fn compute_pulse_cache(lm_max: i32) -> (Vec, Vec) { + let nb = NB_EBANDS; + let rows = (lm_max + 2) as usize; + let mut index = vec![-1i16; nb * rows]; + let mut entries: Vec<(usize, i32, usize)> = Vec::new(); + let mut curr = 0usize; + for i in 0..rows { + for j in 0..nb { + let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize; + index[i * nb + j] = -1; + 'search: for k in 0..=i { + for nn in 0..nb { + if k == i && nn >= j { + break; + } + if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize { + index[i * nb + j] = index[k * nb + nn]; + break 'search; + } + } + } + if index[i * nb + j] == -1 && n != 0 { + let mut k = 0; + while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) { + k += 1; + } + index[i * nb + j] = curr as i16; + entries.push((n, k, curr)); + curr += (k + 1) as usize; + } + } + } + let mut bits = vec![0u8; curr]; + for &(n, k, off) in &entries { + let maxk = get_pulses(k) as usize; + let tmp = required_bits(n, maxk, BITRES); + for j in 1..=k { + bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8; + } + bits[off] = k as u8; + } + (index, bits) +} + +fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec { + let mut out = vec![0i32; maxk + 1]; + if n == 1 { + for o in out.iter_mut().skip(1) { + *o = 1 << frac; + } + } else { + for (k, o) in out.iter_mut().enumerate().skip(1) { + *o = log2_frac(pvq_v(n, k) as u32, frac); + } + } + out +} + +fn compute_caps(m: &Mode) -> Vec { + let nb = NB_EBANDS; + let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb]; + for i in 0..=MAX_LM as i32 { + for c in 1..=2i32 { + for j in 0..nb { + let mut n0 = EBANDS[j + 1] - EBANDS[j]; + let max_bits; + if n0 << i == 1 { + max_bits = c * (1 + MAX_FINE_BITS) << BITRES; + } else { + let mut lm0 = 0; + if n0 > 2 { + n0 >>= 1; + lm0 -= 1; + } else if n0 <= 1 { + lm0 = i.min(1); + n0 <<= lm0; + } + let pcache = m.cache(lm0, j); + let mut mb = pcache[pcache[0] as usize] as i32 + 1; + let mut n = n0; + for k in 0..(i - lm0) { + mb <<= 1; + let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET; + let num = 459 * ((2 * n - 1) * offset + mb); + let den = ((2 * n - 1) << 9) - 459; + let qb = ((num + (den >> 1)) / den).min(57); + mb += qb; + n <<= 1; + } + if c == 2 { + mb <<= 1; + let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET }; + let ndof = 2 * n - 1 - (n == 2) as i32; + let p = if n == 2 { 512 } else { 487 }; + let num = p * (mb + ndof * offset); + let den = (ndof << 9) - p; + let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 }); + mb += qb; + } + let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 }; + let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET; + if n == 2 { + offset += (1 << BITRES) >> 2; + } + let num = mb + ndof * offset; + let den = (ndof - 1) << BITRES; + let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS); + mb += c * qb << BITRES; + max_bits = mb; + } + let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i); + let v = (4 * max_bits / w) - 64; + caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8; + } + } + } + caps +} + +/// Result of the allocation process. +pub struct Allocation { + pub coded_bands: usize, + pub intensity: usize, + pub dual_stereo: bool, + pub balance: i32, + pub pulses: [i32; NB_EBANDS], + pub fine_quant: [i32; NB_EBANDS], + pub fine_priority: [i32; NB_EBANDS], +} + +/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo +/// with no intensity bands, and no band is skipped voluntarily. +#[allow(clippy::too_many_arguments)] +pub fn compute_allocation( + m: &Mode, + start: usize, + end: usize, + offsets: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + alloc_trim: i32, + total: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let lmi = lm as i32; + let mut total = total.max(0); + let mut skip_start = start; + let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= skip_rsv; + let mut intensity_rsv = 0; + let mut dual_stereo_rsv = 0; + if c == 2 { + intensity_rsv = LOG2_FRAC_TABLE[end - start]; + if intensity_rsv > total { + intensity_rsv = 0; + } else { + total -= intensity_rsv; + dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= dual_stereo_rsv; + } + } + let mut bits1 = [0i32; NB_EBANDS]; + let mut bits2 = [0i32; NB_EBANDS]; + let mut thresh = [0i32; NB_EBANDS]; + let mut trim_offset = [0i32; NB_EBANDS]; + for j in start..end { + let w = EBANDS[j + 1] - EBANDS[j]; + thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4); + trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6; + if w << lmi == 1 { + trim_offset[j] -= ci << BITRES; + } + } + let nb_alloc = ALLOC.len() as i32; + let mut lo = 1; + let mut hi = nb_alloc - 1; + loop { + let mut done = false; + let mut psum = 0; + let mid = (lo + hi) >> 1; + for j in (start..end).rev() { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2; + if bitsj > 0 { + bitsj = (bitsj + trim_offset[j]).max(0); + } + bitsj += offsets[j]; + if bitsj >= thresh[j] || done { + done = true; + psum += bitsj.min(cap[j]); + } else if bitsj >= ci << BITRES { + psum += ci << BITRES; + } + } + if psum > total { + hi = mid - 1; + } else { + lo = mid + 1; + } + if lo > hi { + break; + } + } + hi = lo; + lo -= 1; + for j in start..end { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2; + let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 }; + if bits1j > 0 { + bits1j = (bits1j + trim_offset[j]).max(0); + } + if bits2j > 0 { + bits2j = (bits2j + trim_offset[j]).max(0); + } + if lo > 0 { + bits1j += offsets[j]; + } + bits2j += offsets[j]; + if offsets[j] > 0 { + skip_start = j; + } + bits2j = (bits2j - bits1j).max(0); + bits1[j] = bits1j; + bits2[j] = bits2j; + } + interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc) +} + +#[allow(clippy::too_many_arguments)] +fn interp_bits2pulses( + m: &Mode, + start: usize, + end: usize, + skip_start: usize, + bits1: &[i32; NB_EBANDS], + bits2: &[i32; NB_EBANDS], + thresh: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + mut total: i32, + skip_rsv: i32, + mut intensity_rsv: i32, + mut dual_stereo_rsv: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let alloc_floor = ci << BITRES; + let stereo = (c > 1) as i32; + let log_m = (lm as i32) << BITRES; + let mut lo = 0; + let mut hi = 1 << ALLOC_STEPS; + for _ in 0..ALLOC_STEPS { + let mid = (lo + hi) >> 1; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS); + if tmp >= thresh[j] || done { + done = true; + psum += tmp.min(cap[j]); + } else if tmp >= alloc_floor { + psum += alloc_floor; + } + } + if psum > total { + hi = mid; + } else { + lo = mid; + } + } + let mut bits = [0i32; NB_EBANDS]; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS); + if tmp < thresh[j] && !done { + tmp = if tmp >= alloc_floor { alloc_floor } else { 0 }; + } else { + done = true; + } + tmp = tmp.min(cap[j]); + bits[j] = tmp; + psum += tmp; + } + let s = start; + let mut coded_bands = end; + loop { + let j = coded_bands - 1; + if j <= skip_start { + total += skip_rsv; + break; + } + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + let rem = (left - (EBANDS[j] - EBANDS[s])).max(0); + let band_width = EBANDS[coded_bands] - EBANDS[j]; + let band_bits = bits[j] + percoeff * band_width + rem; + if band_bits >= thresh[j].max(alloc_floor + (1 << BITRES)) { + // Keep this band (and every band below it). + enc.bit_logp(true, 1); + break; + } + psum -= bits[j] + intensity_rsv; + if intensity_rsv > 0 { + intensity_rsv = LOG2_FRAC_TABLE[j - s]; + } + psum += intensity_rsv; + if band_bits >= alloc_floor { + psum += alloc_floor; + bits[j] = alloc_floor; + } else { + bits[j] = 0; + } + coded_bands -= 1; + } + let intensity = if intensity_rsv > 0 { + // No intensity stereo: every coded band is coded in full. + enc.uint((coded_bands - s) as u32, (coded_bands + 1 - s) as u32); + coded_bands + } else { + 0 + }; + if intensity <= s { + total += dual_stereo_rsv; + dual_stereo_rsv = 0; + } + let dual_stereo = if dual_stereo_rsv > 0 { + enc.bit_logp(true, 1); + true + } else { + false + }; + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + for j in s..coded_bands { + bits[j] += percoeff * (EBANDS[j + 1] - EBANDS[j]); + } + for j in s..coded_bands { + let tmp = left.min(EBANDS[j + 1] - EBANDS[j]); + bits[j] += tmp; + left -= tmp; + } + let mut ebits = [0i32; NB_EBANDS]; + let mut fine_priority = [0i32; NB_EBANDS]; + let mut balance = 0; + for j in s..coded_bands { + let n0 = EBANDS[j + 1] - EBANDS[j]; + let n = n0 << lm; + let bit = bits[j] + balance; + let mut excess; + if n > 1 { + excess = (bit - cap[j]).max(0); + bits[j] = bit - excess; + let den = ci * n + if c == 2 && n > 2 && !dual_stereo && j < intensity { 1 } else { 0 }; + let nclogn = den * (m.log_n[j] + log_m); + let mut offset = (nclogn >> 1) - den * FINE_OFFSET; + if n == 2 { + offset += (den << BITRES) >> 2; + } + if bits[j] + offset < den * 2 << BITRES { + offset += nclogn >> 2; + } else if bits[j] + offset < den * 3 << BITRES { + offset += nclogn >> 3; + } + let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0); + eb = (eb / den) >> BITRES; + if ci * eb > (bits[j] >> BITRES) { + eb = bits[j] >> stereo >> BITRES; + } + eb = eb.min(MAX_FINE_BITS); + ebits[j] = eb; + fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32; + bits[j] -= ci * eb << BITRES; + } else { + excess = (bit - (ci << BITRES)).max(0); + bits[j] = bit - excess; + ebits[j] = 0; + fine_priority[j] = 1; + } + if excess > 0 { + let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]); + ebits[j] += extra_fine; + let extra_bits = extra_fine * ci << BITRES; + fine_priority[j] = (extra_bits >= excess - balance) as i32; + excess -= extra_bits; + } + balance = excess; + } + for j in coded_bands..end { + ebits[j] = bits[j] >> stereo >> BITRES; + bits[j] = 0; + fine_priority[j] = (ebits[j] < 1) as i32; + } + Allocation { coded_bands, intensity, dual_stereo, balance, pulses: bits, fine_quant: ebits, fine_priority } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caps_match_decoder_reference() { + let m = Mode::new(); + let first_rows: [u8; 42] = [ + 224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, 178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, + 240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, + ]; + assert_eq!(&m.caps[..42], &first_rows[..]); + } + + #[test] + fn pvq_counts() { + assert_eq!(pvq_v(5, 1), 10); + assert_eq!(pvq_v(1, 7), 2); + assert_eq!(pvq_v(2, 9), 36); + assert_eq!(pvq_v(3, 2), 18); + } +} diff --git a/crates/opusenc/src/tables.rs b/crates/opusenc/src/tables.rs new file mode 100644 index 000000000..00d623af0 --- /dev/null +++ b/crates/opusenc/src/tables.rs @@ -0,0 +1,122 @@ +//! Constant tables of the 48 kHz CELT mode (RFC 6716 §4.3, Tables 55–58 and §4.3.2). +//! +//! Transcribed from RFC 6716; copied from FilmCraft's spec-derived `filmcraft-opus` decoder +//! (first-party, MIT OR Apache-2.0) so both sides share the exact same values. + +/// Number of energy bands. +pub const NB_EBANDS: usize = 21; +/// Band edges in units of 2.5 ms MDCT bins (Table 55). +pub const EBANDS: [i32; NB_EBANDS + 1] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100]; +/// Overlap of the low-overlap window (samples at 48 kHz). +pub const OVERLAP: usize = 120; +/// Largest LM (20 ms frames). +pub const MAX_LM: usize = 3; +/// Pre-emphasis coefficient. +pub const PREEMPH: f32 = 0.850_006_1; + +/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin. +pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([ + [0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200], + [0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200], + [0, 75, 90, 103, 112, 120, 130, 138, 148, 158, 200], + [0, 69, 84, 93, 104, 114, 124, 132, 142, 152, 200], + [0, 63, 78, 86, 95, 103, 113, 123, 133, 143, 200], + [0, 56, 71, 80, 89, 97, 107, 117, 127, 137, 200], + [0, 49, 65, 75, 83, 91, 101, 111, 121, 131, 200], + [0, 40, 58, 70, 78, 85, 95, 105, 115, 125, 200], + [0, 34, 51, 65, 72, 78, 88, 98, 108, 118, 198], + [0, 29, 45, 59, 66, 72, 82, 92, 102, 112, 193], + [0, 20, 39, 53, 60, 66, 76, 86, 96, 106, 188], + [0, 18, 32, 47, 54, 60, 70, 80, 90, 100, 183], + [0, 10, 26, 40, 47, 54, 64, 74, 84, 94, 178], + [0, 0, 20, 31, 39, 47, 57, 67, 77, 87, 173], + [0, 0, 12, 23, 32, 41, 51, 61, 71, 81, 168], + [0, 0, 0, 15, 25, 35, 45, 55, 65, 75, 163], + [0, 0, 0, 4, 17, 29, 39, 49, 59, 69, 158], + [0, 0, 0, 0, 12, 23, 33, 43, 53, 63, 153], + [0, 0, 0, 0, 1, 16, 26, 36, 46, 56, 148], + [0, 0, 0, 0, 0, 10, 15, 20, 30, 45, 129], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104], +]); + +const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] { + let mut out = [[0u8; NB_EBANDS]; 11]; + let mut b = 0; + while b < NB_EBANDS { + let mut q = 0; + while q < 11 { + out[q][b] = t[b][q]; + q += 1; + } + b += 1; + } + out +} + +/// Mean band energy (log2 domain) removed before coarse quantisation. +pub const E_MEANS: [f32; 25] = [ + 6.4375, 6.25, 5.75, 5.3125, 5.0625, 4.8125, 4.5, 4.375, 4.875, 4.6875, 4.5625, 4.4375, 4.875, 4.625, 4.3125, 4.5, 4.375, 4.625, 4.75, 4.4375, 3.75, 3.75, + 3.75, 3.75, 3.75, +]; + +/// Inter-frame prediction coefficient (alpha) per LM, Q15. +pub const PRED_COEF: [f32; 4] = [29440.0 / 32768.0, 26112.0 / 32768.0, 21248.0 / 32768.0, 16384.0 / 32768.0]; +/// Inter-band prediction coefficient (beta) per LM for inter frames, Q15. +pub const BETA_COEF: [f32; 4] = [30147.0 / 32768.0, 22282.0 / 32768.0, 12124.0 / 32768.0, 6554.0 / 32768.0]; +/// Beta for intra frames. +pub const BETA_INTRA: f32 = 4915.0 / 32768.0; + +/// Laplace model parameters for coarse energy: `[LM][intra][2*band + {0: prob, 1: decay}]`. +pub const E_PROB_MODEL: [[[u8; 42]; 2]; 4] = [ + [ + [ + 72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, 114, 40, 132, 26, 132, + 26, 145, 17, 161, 12, 176, 10, 177, 11, + ], + [ + 24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, 91, 67, 100, 59, 108, 50, + 120, 40, 122, 37, 97, 43, 78, 50, + ], + ], + [ + [ + 83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, 146, 19, 162, 12, 165, 10, + 178, 7, 189, 6, 190, 8, 177, 9, + ], + [ + 23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, 104, 60, 117, 52, 123, 44, + 138, 35, 133, 31, 97, 38, 77, 45, + ], + ], + [ + [ + 61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, 158, 18, 170, 13, 177, + 10, 187, 8, 192, 6, 175, 9, 159, 10, + ], + [ + 21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, 112, 56, 129, 51, 132, 40, + 150, 33, 140, 29, 98, 35, 77, 42, + ], + ], + [ + [ + 42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, 154, 26, 166, 21, 173, + 16, 184, 13, 184, 10, 150, 13, 139, 15, + ], + [ + 22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, 117, 55, 135, 49, 137, 39, + 157, 32, 145, 29, 97, 33, 77, 40, + ], + ], +]; + +/// Inverse CDF for coarse energy when only a few bits remain. +pub const SMALL_ENERGY_ICDF: [u8; 3] = [2, 1, 0]; +/// Conservative log2 in 1/8 bits of the number of coded bands (intensity reservation). +pub const LOG2_FRAC_TABLE: [i32; 24] = [0, 8, 13, 16, 19, 21, 23, 24, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 37]; +/// TF resolution changes: `[LM][4*transient + 2*tf_select + tf_flag]`. +pub const TF_SELECT_TABLE: [[i8; 8]; 4] = [[0, -1, 0, -1, 0, -1, 0, -1], [0, -1, 0, -2, 1, 0, 1, -1], [0, -2, 0, -3, 2, 0, 1, -1], [0, -2, 0, -3, 3, 0, 1, -1]]; +/// Spreading decision `{7, 2, 21, 2}/32`. +pub const SPREAD_ICDF: [u8; 4] = [25, 23, 2, 0]; +/// Allocation trim (Table 58). +pub const TRIM_ICDF: [u8; 11] = [126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0]; diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 0e9c6947a..409ae7414 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -666,6 +666,9 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// One layer of a layered file (Photoshop) instead of its merged image. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer: Option, /// Interpret Footage ▸ Fields and Pulldown ▸ Separate Fields. #[serde(default, skip_serializing_if = "is_default")] pub fields: FieldOrder, @@ -709,6 +712,7 @@ impl Default for Footage { invert_alpha: false, linear_light: false, data: None, + layer: None, } } } @@ -739,6 +743,18 @@ pub struct Proxy { pub enabled: bool, } +/// A layer of a layered still (a Photoshop document) used as footage +/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceLayer { + /// Layer record index in the file (bottom of the stack = 0). + pub index: u32, + pub name: String, + /// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size. + #[serde(default)] + pub layer_size: bool, +} + fn one() -> u32 { 1 } diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index 31537a5b4..d0f0c91f0 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -184,17 +184,26 @@ pub enum OutputFormat { ExrSequence, /// Animated GIF. Gif, + /// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio. + WebM, + /// Audio only: WAV (16-bit PCM). + Wav, + /// Audio only: AIFF (16-bit PCM). + Aiff, } impl OutputFormat { - pub const ALL: [OutputFormat; 7] = [ + pub const ALL: [OutputFormat; 10] = [ OutputFormat::H264, OutputFormat::ProRes, + OutputFormat::WebM, OutputFormat::PngSequence, OutputFormat::JpegSequence, OutputFormat::TiffSequence, OutputFormat::ExrSequence, OutputFormat::Gif, + OutputFormat::Wav, + OutputFormat::Aiff, ]; pub fn label(self) -> &'static str { @@ -206,6 +215,9 @@ impl OutputFormat { OutputFormat::TiffSequence => "TIFF Sequence", OutputFormat::ExrSequence => "OpenEXR Sequence", OutputFormat::Gif => "Animated GIF", + OutputFormat::WebM => "WebM (VP9 + Opus)", + OutputFormat::Wav => "WAV", + OutputFormat::Aiff => "AIFF", } } pub fn extension(self) -> &'static str { @@ -217,19 +229,26 @@ impl OutputFormat { OutputFormat::TiffSequence => "tif", OutputFormat::ExrSequence => "exr", OutputFormat::Gif => "gif", + OutputFormat::WebM => "webm", + OutputFormat::Wav => "wav", + OutputFormat::Aiff => "aif", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + } + /// Audio-only outputs (no video is rendered). + pub fn is_audio_only(self) -> bool { + matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence) + !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { - self.is_movie() + self.is_movie() || self.is_audio_only() } /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, /// at least 2). @@ -249,6 +268,9 @@ impl OutputFormat { "tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence, "exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence, "gif" | "animatedgif" => OutputFormat::Gif, + "webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM, + "wav" | "wave" => OutputFormat::Wav, + "aif" | "aiff" | "aifc" => OutputFormat::Aiff, _ => return None, }) } diff --git a/crates/psd/Cargo.toml b/crates/psd/Cargo.toml new file mode 100644 index 000000000..49dda6a3b --- /dev/null +++ b/crates/psd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "effectcraft-psd" +description = "Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer, from Adobe's published file format specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +miniz_oxide = "0.8" diff --git a/crates/psd/README.md b/crates/psd/README.md new file mode 100644 index 000000000..e238b9b2f --- /dev/null +++ b/crates/psd/README.md @@ -0,0 +1,17 @@ +# effectcraft-psd + +Reads Photoshop documents (`.psd`, and `.psb` large documents) for File ▸ Import (as footage, as a +composition, or as a composition with layer sizes retained), and writes a minimal subset (used to +generate test fixtures). + +Implemented from Adobe's public **Adobe Photoshop File Formats Specification** (November 2019 +edition): file header, colour mode data, image resources, layer and mask information (layer +records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`, +`lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`, +`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) +and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit; +Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes. + +Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and +justification of the first style run. No Adobe code, sample files or assets were used; test +fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)). diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs new file mode 100644 index 000000000..e121f823d --- /dev/null +++ b/crates/psd/src/descriptor.rs @@ -0,0 +1,430 @@ +//! The descriptor ("action descriptor") structure: typed key/value trees used by text layers, +//! layer effects, fill layers and newer adjustment layers (spec: "Descriptor structure"). + +use crate::read::Reader; +use crate::{Error, Result}; + +/// A descriptor: a class and a list of keyed items (in file order). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Descriptor { + pub class_name: String, + pub class_id: String, + pub items: Vec<(String, DValue)>, +} + +/// One descriptor value. +#[derive(Clone, Debug, PartialEq)] +pub enum DValue { + Descriptor(Descriptor), + List(Vec), + Double(f64), + /// Unit (`#Pxl`, `#Prc`, `#Ang`, `#Pnt`…) and value. + UnitFloat(String, f64), + UnitFloats(String, Vec), + Text(String), + /// Enumerated value: type and value ids. + Enum(String, String), + Integer(i32), + LargeInteger(i64), + Bool(bool), + Class(String, String), + Raw(Vec), + Reference(Vec), +} + +/// An item of a reference (`obj `). +#[derive(Clone, Debug, PartialEq)] +pub enum RefItem { + Property(String, String), + Class(String), + Enum(String, String, String), + Offset(String, i32), + Identifier(i32), + Index(i32), + Name(String, String), +} + +impl Descriptor { + pub fn get(&self, key: &str) -> Option<&DValue> { + self.items.iter().find(|(k, _)| k == key).map(|(_, v)| v) + } + pub fn obj(&self, key: &str) -> Option<&Descriptor> { + match self.get(key)? { + DValue::Descriptor(d) => Some(d), + _ => None, + } + } + /// A number: doubles, unit floats, integers. + pub fn num(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Double(v) | DValue::UnitFloat(_, v) => Some(*v), + DValue::Integer(v) => Some(*v as f64), + DValue::LargeInteger(v) => Some(*v as f64), + _ => None, + } + } + pub fn bool(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Bool(b) => Some(*b), + _ => None, + } + } + pub fn text(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Text(s) => Some(s), + _ => None, + } + } + /// The value id of an enumerated item. + pub fn enum_value(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Enum(_, v) => Some(v), + _ => None, + } + } + pub fn raw(&self, key: &str) -> Option<&[u8]> { + match self.get(key)? { + DValue::Raw(v) => Some(v), + _ => None, + } + } + /// An RGB colour object (`RGBC` with `Rd `/`Grn `/`Bl ` in 0..255) as 0..1 values; also + /// reads grayscale (`Gry `) colours. + pub fn color(&self, key: &str) -> Option<[f64; 3]> { + let c = self.obj(key)?; + if let (Some(r), Some(g), Some(b)) = (c.num("Rd "), c.num("Grn "), c.num("Bl ")) { + return Some([r / 255.0, g / 255.0, b / 255.0]); + } + if let (Some(r), Some(g), Some(b)) = (c.num("redFloat"), c.num("greenFloat"), c.num("blueFloat")) { + return Some([r, g, b]); + } + c.num("Gry ").map(|g| { + let v = 1.0 - g / 100.0; + [v, v, v] + }) + } +} + +/// A key or class id: length-prefixed string, or a 4-character code when the length is 0. +pub(crate) fn read_id(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let n = if n == 0 { 4 } else { n }; + let b = r.bytes(n)?; + Ok(String::from_utf8_lossy(b).into_owned()) +} + +pub(crate) fn read_unicode(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(r.u16()?); + } + while v.last() == Some(&0) { + v.pop(); + } + Ok(String::from_utf16_lossy(&v)) +} + +/// Read a descriptor (after its version field). +pub fn read_descriptor(r: &mut Reader) -> Result { + let class_name = read_unicode(r)?; + let class_id = read_id(r)?; + let n = r.u32()? as usize; + if n > 100_000 { + return Err(Error::Invalid("descriptor item count".into())); + } + let mut items = Vec::with_capacity(n.min(64)); + for _ in 0..n { + let key = read_id(r)?; + let ty = r.tag()?; + let v = read_value(r, &ty, 0)?; + items.push((key, v)); + } + Ok(Descriptor { class_name, class_id, items }) +} + +fn read_value(r: &mut Reader, ty: &[u8; 4], depth: usize) -> Result { + if depth > 64 { + return Err(Error::Invalid("descriptor nesting".into())); + } + Ok(match ty { + b"Objc" | b"GlbO" => DValue::Descriptor(read_descriptor(r)?), + b"VlLs" => { + let n = r.u32()? as usize; + if n > 1_000_000 { + return Err(Error::Invalid("descriptor list length".into())); + } + let mut v = Vec::with_capacity(n.min(256)); + for _ in 0..n { + let t = r.tag()?; + v.push(read_value(r, &t, depth + 1)?); + } + DValue::List(v) + } + b"doub" => DValue::Double(r.f64()?), + b"UntF" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + DValue::UnitFloat(unit, r.f64()?) + } + b"UnFl" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n.min(1024)); + for _ in 0..n { + v.push(r.f64()?); + } + DValue::UnitFloats(unit, v) + } + b"TEXT" => DValue::Text(read_unicode(r)?), + b"enum" => { + let t = read_id(r)?; + DValue::Enum(t, read_id(r)?) + } + b"long" => DValue::Integer(r.u32()? as i32), + b"comp" => DValue::LargeInteger(r.u64()? as i64), + b"bool" => DValue::Bool(r.u8()? != 0), + b"type" | b"GlbC" => { + let name = read_unicode(r)?; + DValue::Class(name, read_id(r)?) + } + b"alis" | b"tdta" | b"Pth " => { + let n = r.u32()? as usize; + DValue::Raw(r.bytes(n)?.to_vec()) + } + b"obj " => { + let n = r.u32()? as usize; + let mut items = Vec::with_capacity(n.min(16)); + for _ in 0..n { + let t = r.tag()?; + items.push(match &t { + b"prop" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Property(class, read_id(r)?) + } + b"Clss" => { + let _name = read_unicode(r)?; + RefItem::Class(read_id(r)?) + } + b"Enmr" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + let ty = read_id(r)?; + RefItem::Enum(class, ty, read_id(r)?) + } + b"rele" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Offset(class, r.u32()? as i32) + } + b"Idnt" => RefItem::Identifier(r.u32()? as i32), + b"indx" => RefItem::Index(r.u32()? as i32), + b"name" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Name(class, read_unicode(r)?) + } + other => return Err(Error::Invalid(format!("reference item `{}`", String::from_utf8_lossy(other)))), + }); + } + DValue::Reference(items) + } + other => return Err(Error::Invalid(format!("descriptor type `{}`", String::from_utf8_lossy(other)))), + }) +} + +// ---------------------------------------------------------------- writing + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +fn put_id(out: &mut Vec, id: &str) { + if id.len() == 4 { + put_u32(out, 0); + } else { + put_u32(out, id.len() as u32); + } + out.extend_from_slice(id.as_bytes()); +} + +pub(crate) fn put_unicode(out: &mut Vec, s: &str) { + let v: Vec = s.encode_utf16().chain(std::iter::once(0)).collect(); + put_u32(out, v.len() as u32); + for c in v { + out.extend_from_slice(&c.to_be_bytes()); + } +} + +/// Serialise a descriptor (without the version field). +pub fn write_descriptor(out: &mut Vec, d: &Descriptor) { + put_unicode(out, &d.class_name); + put_id(out, &d.class_id); + put_u32(out, d.items.len() as u32); + for (k, v) in &d.items { + put_id(out, k); + write_value(out, v); + } +} + +fn write_value(out: &mut Vec, v: &DValue) { + match v { + DValue::Descriptor(d) => { + out.extend_from_slice(b"Objc"); + write_descriptor(out, d); + } + DValue::List(l) => { + out.extend_from_slice(b"VlLs"); + put_u32(out, l.len() as u32); + for x in l { + write_value(out, x); + } + } + DValue::Double(x) => { + out.extend_from_slice(b"doub"); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloat(u, x) => { + out.extend_from_slice(b"UntF"); + out.extend_from_slice(&tag4(u)); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloats(u, xs) => { + out.extend_from_slice(b"UnFl"); + out.extend_from_slice(&tag4(u)); + put_u32(out, xs.len() as u32); + for x in xs { + out.extend_from_slice(&x.to_be_bytes()); + } + } + DValue::Text(s) => { + out.extend_from_slice(b"TEXT"); + put_unicode(out, s); + } + DValue::Enum(t, e) => { + out.extend_from_slice(b"enum"); + put_id(out, t); + put_id(out, e); + } + DValue::Integer(i) => { + out.extend_from_slice(b"long"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::LargeInteger(i) => { + out.extend_from_slice(b"comp"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::Bool(b) => { + out.extend_from_slice(b"bool"); + out.push(*b as u8); + } + DValue::Class(n, c) => { + out.extend_from_slice(b"type"); + put_unicode(out, n); + put_id(out, c); + } + DValue::Raw(b) => { + out.extend_from_slice(b"tdta"); + put_u32(out, b.len() as u32); + out.extend_from_slice(b); + } + DValue::Reference(items) => { + out.extend_from_slice(b"obj "); + put_u32(out, items.len() as u32); + for it in items { + match it { + RefItem::Property(c, k) => { + out.extend_from_slice(b"prop"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, k); + } + RefItem::Class(c) => { + out.extend_from_slice(b"Clss"); + put_unicode(out, ""); + put_id(out, c); + } + RefItem::Enum(c, t, e) => { + out.extend_from_slice(b"Enmr"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, t); + put_id(out, e); + } + RefItem::Offset(c, o) => { + out.extend_from_slice(b"rele"); + put_unicode(out, ""); + put_id(out, c); + out.extend_from_slice(&o.to_be_bytes()); + } + RefItem::Identifier(i) => { + out.extend_from_slice(b"Idnt"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Index(i) => { + out.extend_from_slice(b"indx"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Name(c, n) => { + out.extend_from_slice(b"name"); + put_unicode(out, ""); + put_id(out, c); + put_unicode(out, n); + } + } + } + } + } +} + +fn tag4(s: &str) -> [u8; 4] { + let mut t = *b" "; + for (i, b) in s.bytes().take(4).enumerate() { + t[i] = b; + } + t +} + +/// Builder helpers for descriptors (tests and the writer). +impl Descriptor { + pub fn new(class_id: &str) -> Descriptor { + Descriptor { class_name: String::new(), class_id: class_id.into(), items: vec![] } + } + pub fn with(mut self, key: &str, v: DValue) -> Descriptor { + self.items.push((key.into(), v)); + self + } + /// An `RGBC` colour object from 0..1 values. + pub fn rgb(c: [f64; 3]) -> Descriptor { + Descriptor::new("RGBC").with("Rd ", DValue::Double(c[0] * 255.0)).with("Grn ", DValue::Double(c[1] * 255.0)).with("Bl ", DValue::Double(c[2] * 255.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let d = Descriptor::new("null") + .with("Txt ", DValue::Text("Héllo".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 0.5, 0.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 75.0)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("enab", DValue::Bool(true)) + .with("longKeyName", DValue::List(vec![DValue::Integer(3), DValue::Double(1.5)])) + .with("EngineData", DValue::Raw(b"<< >>".to_vec())) + .with("ref", DValue::Reference(vec![RefItem::Enum("Lyr ".into(), "Ordn".into(), "Trgt".into()), RefItem::Index(2)])); + let mut out = vec![]; + write_descriptor(&mut out, &d); + let back = read_descriptor(&mut Reader::new(&out)).unwrap(); + assert_eq!(back.class_id, "null"); + assert_eq!(back.text("Txt "), Some("Héllo")); + assert_eq!(back.color("Clr "), Some([1.0, 0.5, 0.0])); + assert_eq!(back.num("Opct"), Some(75.0)); + assert_eq!(back.enum_value("Md "), Some("Mltp")); + assert_eq!(back.bool("enab"), Some(true)); + assert_eq!(back.raw("EngineData"), Some(&b"<< >>"[..])); + assert_eq!(back, d); + } +} diff --git a/crates/psd/src/engine_data.rs b/crates/psd/src/engine_data.rs new file mode 100644 index 000000000..c4f230380 --- /dev/null +++ b/crates/psd/src/engine_data.rs @@ -0,0 +1,288 @@ +//! The text engine data of type layers (`EngineData` in the `TySh` descriptor): a PostScript-like +//! tree of dictionaries (`<< /Key value >>`), arrays (`[ ]`), numbers, names, booleans and +//! strings (`(þÿ` + UTF-16BE with backslash escapes `)`). Read best-effort. + +#[derive(Clone, Debug, PartialEq)] +pub enum EValue { + Dict(Vec<(String, EValue)>), + Array(Vec), + Num(f64), + Bool(bool), + Name(String), + Str(String), +} + +impl EValue { + pub fn get(&self, key: &str) -> Option<&EValue> { + match self { + EValue::Dict(v) => v.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + pub fn idx(&self, i: usize) -> Option<&EValue> { + match self { + EValue::Array(v) => v.get(i), + _ => None, + } + } + /// Follow a `/`-separated path; numeric segments index arrays. + pub fn path(&self, p: &str) -> Option<&EValue> { + let mut cur = self; + for seg in p.split('/').filter(|s| !s.is_empty()) { + cur = match seg.parse::() { + Ok(i) => cur.idx(i)?, + Err(_) => cur.get(seg)?, + }; + } + Some(cur) + } + pub fn num(&self) -> Option { + match self { + EValue::Num(n) => Some(*n), + _ => None, + } + } + pub fn str(&self) -> Option<&str> { + match self { + EValue::Str(s) | EValue::Name(s) => Some(s), + _ => None, + } + } +} + +struct P<'a> { + b: &'a [u8], + i: usize, +} + +impl P<'_> { + fn ws(&mut self) { + while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == 0) { + self.i += 1; + } + } + fn peek(&self, s: &[u8]) -> bool { + self.b[self.i..].starts_with(s) + } + fn value(&mut self, depth: usize) -> Option { + if depth > 128 { + return None; + } + self.ws(); + if self.i >= self.b.len() { + return None; + } + if self.peek(b"<<") { + self.i += 2; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Dict(v)); + } + if self.peek(b">>") { + self.i += 2; + return Some(EValue::Dict(v)); + } + if self.b[self.i] != b'/' { + // Unexpected token: skip it. + self.i += 1; + continue; + } + let key = self.name(); + let val = self.value(depth + 1)?; + v.push((key, val)); + } + } + match self.b[self.i] { + b'[' => { + self.i += 1; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Array(v)); + } + if self.b[self.i] == b']' { + self.i += 1; + return Some(EValue::Array(v)); + } + let start = self.i; + match self.value(depth + 1) { + Some(x) => v.push(x), + None => return Some(EValue::Array(v)), + } + if self.i == start { + self.i += 1; + } + } + } + b'/' => Some(EValue::Name(self.name())), + b'(' => Some(EValue::Str(self.string())), + _ => { + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + let tok = std::str::from_utf8(&self.b[start..self.i]).unwrap_or(""); + if self.i == start { + self.i += 1; + return Some(EValue::Name(String::new())); + } + Some(match tok { + "true" => EValue::Bool(true), + "false" => EValue::Bool(false), + t => t.parse::().map(EValue::Num).unwrap_or_else(|_| EValue::Name(t.to_string())), + }) + } + } + } + fn name(&mut self) -> String { + self.i += 1; // '/' + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + String::from_utf8_lossy(&self.b[start..self.i]).into_owned() + } + fn string(&mut self) -> String { + self.i += 1; // '(' + let mut raw = vec![]; + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'\\' if self.i < self.b.len() => { + let e = self.b[self.i]; + self.i += 1; + raw.push(match e { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + other => other, + }); + } + b')' => break, + _ => raw.push(c), + } + } + if raw.starts_with(&[0xFE, 0xFF]) { + let u: Vec = raw[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u).trim_end_matches('\0').to_string() + } else { + String::from_utf8_lossy(&raw).into_owned() + } + } +} + +/// Parse engine data (returns an empty dictionary on garbage). +pub fn parse(b: &[u8]) -> EValue { + let mut p = P { b, i: 0 }; + p.value(0).unwrap_or(EValue::Dict(vec![])) +} + +/// Encode a string the way engine data stores it (UTF-16BE with a BOM, escaped). +pub fn encode_string(s: &str) -> Vec { + let mut out = vec![b'(', 0xFE, 0xFF]; + for u in s.encode_utf16() { + for b in u.to_be_bytes() { + if matches!(b, b'(' | b')' | b'\\') { + out.push(b'\\'); + } + out.push(b); + } + } + out.push(b')'); + out +} + +/// The first style run's character attributes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextStyle { + pub font: Option, + pub size: Option, + /// RGB 0..1. + pub color: Option<[f64; 3]>, + /// 0 left, 1 right, 2 centre, 3+ justified variants. + pub justification: Option, + pub tracking: Option, + pub leading: Option, + pub faux_bold: bool, + pub faux_italic: bool, + /// Paragraph (box) text rather than point text. + pub box_text: bool, +} + +pub fn text_style(ed: &EValue) -> TextStyle { + let engine = ed.get("EngineDict").unwrap_or(ed); + let resources = ed.get("ResourceDict").or_else(|| ed.get("DocumentResources")); + let run = engine.path("StyleRun/RunArray/0/StyleSheet/StyleSheetData"); + let normal = resources.and_then(|r| r.path("StyleSheetSet/0/StyleSheetData")); + let attr = |k: &str| run.and_then(|r| r.get(k)).or_else(|| normal.and_then(|n| n.get(k))); + let font = + attr("Font").and_then(EValue::num).and_then(|i| resources?.path(&format!("FontSet/{}/Name", i as usize))).and_then(EValue::str).map(str::to_string); + let color = attr("FillColor").and_then(|c| c.get("Values")).and_then(|v| match v { + EValue::Array(a) if a.len() >= 4 => Some([a[1].num()?, a[2].num()?, a[3].num()?]), + _ => None, + }); + let para = engine.path("ParagraphRun/RunArray/0/ParagraphSheet/Properties"); + let para_normal = resources.and_then(|r| r.path("ParagraphSheetSet/0/Properties")); + let justification = + para.and_then(|p| p.get("Justification")).or_else(|| para_normal.and_then(|p| p.get("Justification"))).and_then(EValue::num).map(|n| n as u8); + let auto_leading = attr("AutoLeading").is_none_or(|v| *v != EValue::Bool(false)); + let box_text = engine.path("Rendered/Shapes/Children/0/ShapeType").and_then(EValue::num) == Some(1.0); + TextStyle { + font, + size: attr("FontSize").and_then(EValue::num), + color, + justification, + tracking: attr("Tracking").and_then(EValue::num), + leading: if auto_leading { None } else { attr("Leading").and_then(EValue::num) }, + faux_bold: attr("FauxBold") == Some(&EValue::Bool(true)), + faux_italic: attr("FauxItalic") == Some(&EValue::Bool(true)), + box_text, + } +} + +/// Minimal engine data for a type layer with one style run (fixtures, and what a writer needs). +pub fn minimal(text: &str, font: &str, size: f64, color: [f64; 3], justification: u8) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(b"\n\n<<\n\t/EngineDict\n\t<<\n\t\t/Editor << /Text "); + out.extend_from_slice(&encode_string(&format!("{text}\r"))); + out.extend_from_slice(b" >>\n\t\t/ParagraphRun << /RunArray [ << /ParagraphSheet << /Properties << /Justification "); + out.extend_from_slice(justification.to_string().as_bytes()); + out.extend_from_slice(b" >> >> >> ] >>\n\t\t/StyleRun << /RunArray [ << /StyleSheet << /StyleSheetData << /Font 0 /FontSize "); + out.extend_from_slice(format!("{size:.1}").as_bytes()); + out.extend_from_slice( + format!(" /FillColor << /Type 1 /Values [ 1.0 {:.5} {:.5} {:.5} ] >> /AutoLeading true >> >> >> ] >>\n\t>>\n", color[0], color[1], color[2]).as_bytes(), + ); + out.extend_from_slice(b"\t/ResourceDict << /FontSet [ << /Name "); + out.extend_from_slice(&encode_string(font)); + out.extend_from_slice(b" /Type 0 >> ] >>\n>>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_minimal() { + let b = minimal("Hi (there)", "Arial-BoldMT", 36.0, [1.0, 0.5, 0.0], 2); + let v = parse(&b); + assert_eq!(v.path("EngineDict/Editor/Text").and_then(EValue::str), Some("Hi (there)\r")); + let st = text_style(&v); + assert_eq!(st.font.as_deref(), Some("Arial-BoldMT")); + assert_eq!(st.size, Some(36.0)); + assert_eq!(st.color, Some([1.0, 0.5, 0.0])); + assert_eq!(st.justification, Some(2)); + assert!(!st.box_text); + } + + #[test] + fn garbage_is_harmless() { + let _ = parse(b"<< /A [ 1 2 << /B (unterminated"); + let _ = parse(b"]]]>>>> /x"); + let _ = parse(&[0xFF; 64]); + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs new file mode 100644 index 000000000..885ab3c21 --- /dev/null +++ b/crates/psd/src/lib.rs @@ -0,0 +1,965 @@ +//! Photoshop document (PSD / PSB) reader, implemented from Adobe's public *Photoshop File Formats +//! Specification* (see README.md), and a minimal writer ([`write`]) for fixtures. +//! +//! [`Psd::parse`] reads the structure (header, layer records and their additional information, +//! the layer tree) without decoding pixels; [`Psd::composite`] decodes the merged image and +//! [`Psd::layer_pixels`] one layer (with its layer mask applied to alpha) on demand. Pixels come +//! out as straight (unpremultiplied) RGBA `f32` in 0..1, display-encoded (32-bit documents, +//! which are linear, are encoded to sRGB). + +pub mod descriptor; +pub mod engine_data; +mod read; +pub mod write; + +use std::sync::Arc; + +pub use descriptor::{DValue, Descriptor}; +use read::Reader; + +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + NotPsd, + Truncated, + Unsupported(String), + Invalid(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotPsd => write!(f, "not a Photoshop document"), + Error::Truncated => write!(f, "truncated Photoshop document"), + Error::Unsupported(s) => write!(f, "unsupported Photoshop feature: {s}"), + Error::Invalid(s) => write!(f, "invalid Photoshop document: {s}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `bytes` start like a Photoshop document. +pub fn is_psd(bytes: &[u8]) -> bool { + bytes.len() >= 6 && &bytes[..4] == b"8BPS" && matches!(u16::from_be_bytes([bytes[4], bytes[5]]), 1 | 2) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColorMode { + Bitmap, + Grayscale, + Indexed, + Rgb, + Cmyk, + Multichannel, + Duotone, + Lab, +} + +impl ColorMode { + pub fn from_u16(v: u16) -> Option { + Some(match v { + 0 => ColorMode::Bitmap, + 1 => ColorMode::Grayscale, + 2 => ColorMode::Indexed, + 3 => ColorMode::Rgb, + 4 => ColorMode::Cmyk, + 7 => ColorMode::Multichannel, + 8 => ColorMode::Duotone, + 9 => ColorMode::Lab, + _ => return None, + }) + } + pub fn to_u16(self) -> u16 { + match self { + ColorMode::Bitmap => 0, + ColorMode::Grayscale => 1, + ColorMode::Indexed => 2, + ColorMode::Rgb => 3, + ColorMode::Cmyk => 4, + ColorMode::Multichannel => 7, + ColorMode::Duotone => 8, + ColorMode::Lab => 9, + } + } + /// Number of colour channels (before alpha). + pub fn color_channels(self) -> usize { + match self { + ColorMode::Rgb | ColorMode::Lab => 3, + ColorMode::Cmyk => 4, + _ => 1, + } + } +} + +/// A rectangle in document pixels (`right`/`bottom` exclusive). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Rect { + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Rect { + pub fn new(left: i32, top: i32, width: i32, height: i32) -> Rect { + Rect { top, left, bottom: top + height, right: left + width } + } + pub fn width(&self) -> u32 { + (self.right - self.left).max(0) as u32 + } + pub fn height(&self) -> u32 { + (self.bottom - self.top).max(0) as u32 + } + pub fn is_empty(&self) -> bool { + self.width() == 0 || self.height() == 0 + } +} + +/// One channel of a layer: id (0.. colour, -1 transparency, -2 layer mask, -3 real user mask) and +/// where its data (compression + image data) sits in the file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChannelInfo { + pub id: i16, + pub offset: usize, + pub len: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LayerMask { + pub rect: Rect, + /// Value outside the mask rectangle (0 or 255). + pub default_color: u8, + pub disabled: bool, + pub inverted: bool, +} + +/// Layer kind from the section divider setting (`lsct`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Section { + #[default] + Layer, + OpenFolder, + ClosedFolder, + /// The hidden "" record that closes a group (below its children). + Divider, +} + +/// A type layer (`TySh`). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextInfo { + /// Text with `\n` line breaks. + pub text: String, + /// Affine transform (xx, xy, yx, yy, tx, ty): the text origin sits at (tx, ty). + pub transform: [f64; 6], + pub style: engine_data::TextStyle, + /// Text bounds (left, top, right, bottom) relative to the origin, when stored. + pub bounds: Option<[f64; 4]>, +} + +/// Adjustment layers (best effort). +#[derive(Clone, Debug, PartialEq)] +pub enum Adjustment { + Invert, + BrightnessContrast { + brightness: f64, + contrast: f64, + legacy: bool, + }, + HueSaturation { + hue: f64, + saturation: f64, + lightness: f64, + colorize: bool, + }, + Levels { + in_black: f64, + in_white: f64, + gamma: f64, + out_black: f64, + out_white: f64, + }, + Exposure { + exposure: f64, + offset: f64, + gamma: f64, + }, + /// Another adjustment (its tagged-block key, e.g. `curv`, `blnc`). + Other(String), +} + +/// A vector mask path: sub-paths of (in tangent, vertex, out tangent) points in document pixels. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct VectorMask { + pub inverted: bool, + pub disabled: bool, + pub subpaths: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SubPath { + pub closed: bool, + /// Per knot: [control point preceding, anchor, control point leaving] as (x, y). + pub knots: Vec<[[f64; 2]; 3]>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Layer { + /// Index in [`Psd::layers`] (file order: bottom of the stack first). + pub index: usize, + pub name: String, + pub id: Option, + pub rect: Rect, + /// Blend mode key (`norm`, `mul `, `pass`…). + pub blend: [u8; 4], + pub opacity: u8, + /// Fill opacity (`iOpa`), 255 when absent. + pub fill_opacity: u8, + /// Clipped to the layer below. + pub clipping: bool, + pub hidden: bool, + pub protect_transparency: bool, + pub section: Section, + pub channels: Vec, + pub mask: Option, + pub text: Option, + pub adjustment: Option, + /// Solid colour fill layer (`SoCo`), RGB 0..1. + pub fill: Option<[f64; 3]>, + /// Layer effects (`lfx2` descriptor). + pub effects: Option, + pub vector_mask: Option, + /// Keys of every additional-information block the layer carries. + pub keys: Vec<[u8; 4]>, +} + +impl Layer { + pub fn is_group(&self) -> bool { + matches!(self.section, Section::OpenFolder | Section::ClosedFolder) + } + pub fn blend_key(&self) -> &str { + std::str::from_utf8(&self.blend).unwrap_or("norm") + } + pub fn has_pixels(&self) -> bool { + !self.rect.is_empty() && self.channels.iter().any(|c| c.id >= 0) + } +} + +/// The layer tree, top of the stack first. +#[derive(Clone, Debug, PartialEq)] +pub enum Node { + Layer(usize), + Group { layer: usize, children: Vec }, +} + +/// Decoded pixels: straight RGBA, row-major. +#[derive(Clone, Debug, PartialEq)] +pub struct Pixels { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, +} + +/// A parsed Photoshop document. +#[derive(Clone, Debug)] +pub struct Psd { + /// PSB (large document format). + pub psb: bool, + pub width: u32, + pub height: u32, + pub channels: u16, + pub depth: u16, + pub mode: ColorMode, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// The merged image's first extra channel is transparency. + pub merged_alpha: bool, + /// Indexed colour palette (256 R, then G, then B). + pub palette: Vec, + data: Arc<[u8]>, + merged_offset: usize, +} + +impl Psd { + pub fn parse(data: impl Into>) -> Result { + let data: Arc<[u8]> = data.into(); + let mut r = Reader::new(&data); + if r.bytes(4).ok() != Some(b"8BPS") { + return Err(Error::NotPsd); + } + let version = r.u16()?; + let psb = match version { + 1 => false, + 2 => true, + _ => return Err(Error::NotPsd), + }; + r.skip(6)?; + let channels = r.u16()?; + let height = r.u32()?; + let width = r.u32()?; + let depth = r.u16()?; + let mode_v = r.u16()?; + let mode = ColorMode::from_u16(mode_v).ok_or_else(|| Error::Unsupported(format!("colour mode {mode_v}")))?; + if !matches!(depth, 1 | 8 | 16 | 32) { + return Err(Error::Unsupported(format!("{depth}-bit"))); + } + if width == 0 || height == 0 || width > 300_000 || height > 300_000 { + return Err(Error::Invalid(format!("size {width}x{height}"))); + } + // Colour mode data. + let n = r.u32()? as usize; + let palette = r.bytes(n)?.to_vec(); + // Image resources. + let n = r.u32()? as usize; + r.skip(n)?; + // Layer and mask information. + let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 }; + let lm_len = r.length(psb)?; + let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?; + if lm_len > 0 { + let li_len = r.length(psb)?; + let li_end = r.pos.checked_add(li_len).filter(|&e| e <= lm_end).ok_or(Error::Truncated)?; + if li_len > 0 { + psd.layer_info(&data, r.pos, li_end)?; + } + r.pos = li_end; + if r.pos + 4 <= lm_end { + let g = r.u32()? as usize; + r.pos = (r.pos + g).min(lm_end); + } + // Global additional layer information (16/32-bit layer info lives here). + while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) { + if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start { + // The block holds the layer info, normally without its length field; accept + // both layouts. + if psd.layer_info(&data, start, end).is_err() || psd.layers.is_empty() { + psd.layers.clear(); + let skip = if psb { 8 } else { 4 }; + if start + skip < end && psd.layer_info(&data, start + skip, end).is_err() { + psd.layers.clear(); + } + } + } + r.pos = end; + } + } + psd.merged_offset = lm_end; + Ok(psd) + } + + fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> { + let mut r = Reader::at(&data[..end], start); + let count = r.i16()?; + self.merged_alpha = count < 0; + let n = count.unsigned_abs() as usize; + let mut layers = Vec::with_capacity(n); + for i in 0..n { + layers.push(self.layer_record(&mut r, i)?); + } + // Channel image data follows in record order. + for l in &mut layers { + for c in &mut l.channels { + c.offset = r.pos; + r.skip(c.len)?; + } + } + self.layers = layers; + Ok(()) + } + + fn layer_record(&self, r: &mut Reader, index: usize) -> Result { + let psb = self.psb; + let (top, left, bottom, right) = (r.i32()?, r.i32()?, r.i32()?, r.i32()?); + let nch = r.u16()? as usize; + if nch > 64 { + return Err(Error::Invalid("channel count".into())); + } + let mut channels = Vec::with_capacity(nch); + for _ in 0..nch { + let id = r.i16()?; + let len = r.length(psb)?; + channels.push(ChannelInfo { id, offset: 0, len }); + } + if r.bytes(4)? != b"8BIM" { + return Err(Error::Invalid("layer record signature".into())); + } + let blend = r.tag()?; + let opacity = r.u8()?; + let clipping = r.u8()? != 0; + let flags = r.u8()?; + r.skip(1)?; + let extra = r.u32()? as usize; + let extra_end = r.pos.checked_add(extra).filter(|&e| e <= r.data.len()).ok_or(Error::Truncated)?; + let mut l = Layer { + index, + rect: Rect { top, left, bottom, right }, + blend, + opacity, + fill_opacity: 255, + clipping, + hidden: flags & 2 != 0, + protect_transparency: flags & 1 != 0, + channels, + ..Default::default() + }; + // Layer mask data. + let mlen = r.u32()? as usize; + let mstart = r.pos; + if mlen >= 18 { + let rect = Rect { top: r.i32()?, left: r.i32()?, bottom: r.i32()?, right: r.i32()? }; + let default_color = r.u8()?; + let mflags = r.u8()?; + l.mask = Some(LayerMask { rect, default_color, disabled: mflags & 2 != 0, inverted: mflags & 4 != 0 }); + } + r.pos = mstart + mlen; + // Blending ranges. + let blen = r.u32()? as usize; + r.skip(blen)?; + l.name = r.pascal(4)?; + let mut pos = r.pos; + let data = r.data; + while let Some((key, start, end)) = next_block(data, &mut pos, extra_end, psb) { + l.keys.push(key); + let block = &data[start..end]; + // A malformed block only loses that block's information. + let _ = self.layer_block(&mut l, &key, block); + pos = end; + } + r.pos = extra_end; + Ok(l) + } + + fn layer_block(&self, l: &mut Layer, key: &[u8; 4], b: &[u8]) -> Result<()> { + let mut r = Reader::new(b); + match key { + b"luni" => { + let name = descriptor::read_unicode(&mut r)?; + if !name.is_empty() { + l.name = name; + } + } + b"lyid" => l.id = Some(r.u32()?), + b"iOpa" => l.fill_opacity = r.u8()?, + b"lsct" | b"lsdk" => { + l.section = match r.u32()? { + 1 => Section::OpenFolder, + 2 => Section::ClosedFolder, + 3 => Section::Divider, + _ => Section::Layer, + }; + if b.len() >= 12 && &b[4..8] == b"8BIM" { + l.blend.copy_from_slice(&b[8..12]); + } + } + b"TySh" => { + let _ver = r.u16()?; + let mut t = [0.0; 6]; + for v in &mut t { + *v = r.f64()?; + } + let _text_ver = r.u16()?; + let _desc_ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + let text = d.text("Txt ").unwrap_or_default().replace("\r\n", "\n").replace('\r', "\n"); + let text = text.strip_suffix('\n').map(str::to_string).unwrap_or(text); + let style = d.raw("EngineData").map(|e| engine_data::text_style(&engine_data::parse(e))).unwrap_or_default(); + let bounds = + d.obj("bounds").or_else(|| d.obj("boundingBox")).and_then(|o| Some([o.num("Left")?, o.num("Top ")?, o.num("Rght")?, o.num("Btom")?])); + l.text = Some(TextInfo { text, transform: t, style, bounds }); + } + b"lfx2" => { + let _ver = r.u32()?; + let _dver = r.u32()?; + l.effects = Some(descriptor::read_descriptor(&mut r)?); + } + b"SoCo" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.fill = d.color("Clr "); + } + b"nvrt" => l.adjustment = Some(Adjustment::Invert), + b"brit" => { + let brightness = r.i16()? as f64; + let contrast = r.i16()? as f64; + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::BrightnessContrast { brightness, contrast, legacy: true }); + } + } + b"CgEd" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.adjustment = Some(Adjustment::BrightnessContrast { + brightness: d.num("Brgh").unwrap_or(0.0), + contrast: d.num("Cntr").unwrap_or(0.0), + legacy: d.bool("useLegacy").unwrap_or(false), + }); + } + b"hue2" | b"hue " => { + let _ver = r.u16()?; + let colorize = r.u8()? != 0; + r.skip(1)?; + let ch = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let m = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let (hue, saturation, lightness) = if colorize { ch } else { m }; + l.adjustment = Some(Adjustment::HueSaturation { hue, saturation, lightness, colorize }); + } + b"levl" => { + let _ver = r.u16()?; + let (ib, iw, ob, ow, g) = (r.i16()?, r.i16()?, r.i16()?, r.i16()?, r.i16()?); + l.adjustment = + Some(Adjustment::Levels { in_black: ib as f64, in_white: iw as f64, gamma: g as f64 / 100.0, out_black: ob as f64, out_white: ow as f64 }); + } + b"expA" => { + let _ver = r.u16()?; + let exposure = r.f32()? as f64; + let offset = r.f32()? as f64; + let gamma = r.f32()? as f64; + l.adjustment = Some(Adjustment::Exposure { exposure, offset, gamma }); + } + b"curv" | b"blnc" | b"post" | b"thrs" | b"selc" | b"mixr" | b"phfl" | b"vibA" | b"blwh" | b"grdm" | b"clrL" => { + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned())); + } + } + b"vmsk" | b"vsms" => { + let _ver = r.u32()?; + let flags = r.u32()?; + l.vector_mask = Some(self.vector_paths(&mut r, flags)?); + } + _ => {} + } + Ok(()) + } + + fn vector_paths(&self, r: &mut Reader, flags: u32) -> Result { + let (w, h) = (self.width as f64, self.height as f64); + let mut vm = VectorMask { inverted: flags & 1 != 0, disabled: flags & 4 != 0, subpaths: vec![] }; + let fixed = |r: &mut Reader| -> Result { Ok(r.i32()? as f64 / (1 << 24) as f64) }; + while r.remaining() >= 26 { + let sel = r.u16()?; + match sel { + 0 | 3 => { + let _n = r.u16()?; + r.skip(22)?; + vm.subpaths.push(SubPath { closed: sel == 0, knots: vec![] }); + } + 1 | 2 | 4 | 5 => { + let mut pts = [[0.0; 2]; 3]; + for p in &mut pts { + let y = fixed(r)?; + let x = fixed(r)?; + *p = [x * w, y * h]; + } + if let Some(sp) = vm.subpaths.last_mut() { + sp.knots.push(pts); + } + } + _ => r.skip(24)?, + } + } + vm.subpaths.retain(|s| !s.knots.is_empty()); + Ok(vm) + } + + /// The layer tree, top of the stack first. + pub fn tree(&self) -> Vec { + let mut stack: Vec> = vec![vec![]]; + for l in &self.layers { + match l.section { + Section::Divider => stack.push(vec![]), + Section::OpenFolder | Section::ClosedFolder => { + let children = if stack.len() > 1 { stack.pop().unwrap_or_default() } else { vec![] }; + if let Some(top) = stack.last_mut() { + top.push(Node::Group { layer: l.index, children }); + } + } + Section::Layer => { + if let Some(top) = stack.last_mut() { + top.push(Node::Layer(l.index)); + } + } + } + } + while stack.len() > 1 { + let c = stack.pop().unwrap_or_default(); + if let Some(top) = stack.last_mut() { + top.extend(c); + } + } + fn rev(v: &mut Vec) { + v.reverse(); + for n in v { + if let Node::Group { children, .. } = n { + rev(children); + } + } + } + let mut out = stack.pop().unwrap_or_default(); + rev(&mut out); + out + } + + fn bytes_per_sample(&self) -> usize { + match self.depth { + 16 => 2, + 32 => 4, + _ => 1, + } + } + + /// Decode one channel of `w`×`h` samples at `offset` (compression field first). Values 0..1. + fn decode_channel(&self, offset: usize, len: usize, w: usize, h: usize) -> Result> { + let end = offset.checked_add(len).filter(|&e| e <= self.data.len()).ok_or(Error::Truncated)?; + let mut r = Reader::at(&self.data[..end], offset); + let comp = r.u16()?; + let rowb = self.row_bytes(w); + let raw = match comp { + 0 => r.bytes(rowb * h)?.to_vec(), + 1 => { + let mut counts = Vec::with_capacity(h); + for _ in 0..h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + let mut out = Vec::with_capacity(rowb * h); + for c in counts { + unpack_bits(r.bytes(c)?, rowb, &mut out)?; + } + out + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let mut v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + v.resize(rowb * h, 0); + if comp == 3 { + self.unpredict(&mut v, w, h); + } + v + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + }; + Ok(self.samples(&raw, w, h)) + } + + fn row_bytes(&self, w: usize) -> usize { + if self.depth == 1 { w.div_ceil(8) } else { w * self.bytes_per_sample() } + } + + fn unpredict(&self, v: &mut [u8], w: usize, h: usize) { + match self.depth { + 16 => { + for row in v.chunks_exact_mut(w * 2).take(h) { + let mut prev = 0u16; + for c in row.chunks_exact_mut(2) { + let x = u16::from_be_bytes([c[0], c[1]]).wrapping_add(prev); + c.copy_from_slice(&x.to_be_bytes()); + prev = x; + } + } + } + 32 => { + for row in v.chunks_exact_mut(w * 4).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + // Bytes are stored plane by plane (all first bytes, then all second bytes…). + let planar = row.to_vec(); + for x in 0..w { + for b in 0..4 { + row[x * 4 + b] = planar[b * w + x]; + } + } + } + } + _ => { + for row in v.chunks_exact_mut(w).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + } + } + } + } + + fn samples(&self, raw: &[u8], w: usize, h: usize) -> Vec { + match self.depth { + 1 => { + let rowb = w.div_ceil(8); + let mut out = Vec::with_capacity(w * h); + for y in 0..h { + for x in 0..w { + let b = raw.get(y * rowb + x / 8).copied().unwrap_or(0); + // 1 = black. + out.push(if b & (0x80 >> (x % 8)) != 0 { 0.0 } else { 1.0 }); + } + } + out + } + 16 => raw.chunks_exact(2).take(w * h).map(|c| u16::from_be_bytes([c[0], c[1]]) as f32 / 65535.0).collect(), + 32 => raw.chunks_exact(4).take(w * h).map(|c| f32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(), + _ => raw.iter().take(w * h).map(|&b| b as f32 / 255.0).collect(), + } + } + + /// Convert colour channels (in the document's mode) to RGB. + fn to_rgb(&self, ch: &[Option>], i: usize) -> [f32; 3] { + let g = |k: usize| ch.get(k).and_then(|c| c.as_ref()).map_or(0.0, |c| c[i]); + let rgb = match self.mode { + ColorMode::Rgb => [g(0), g(1), g(2)], + ColorMode::Cmyk => { + // Stored inverted (1 = no ink). + let k = g(3); + [g(0) * k, g(1) * k, g(2) * k] + } + ColorMode::Indexed if self.palette.len() >= 768 => { + let idx = (g(0) * 255.0).round() as usize & 255; + [self.palette[idx] as f32 / 255.0, self.palette[256 + idx] as f32 / 255.0, self.palette[512 + idx] as f32 / 255.0] + } + ColorMode::Lab => lab_to_rgb(g(0) * 100.0, g(1) * 255.0 - 128.0, g(2) * 255.0 - 128.0), + _ => [g(0); 3], + }; + if self.depth == 32 && self.mode != ColorMode::Lab { rgb.map(srgb_encode) } else { rgb } + } + + /// The merged (flattened) image. + pub fn composite(&self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let mut r = Reader::at(&self.data, self.merged_offset); + let comp = r.u16()?; + let nch = self.channels as usize; + let cc = self.mode.color_channels(); + let want = (cc + usize::from(self.merged_alpha)).min(nch); + let rowb = self.row_bytes(w); + let mut planes: Vec>> = vec![None; nch]; + match comp { + 0 => { + for p in planes.iter_mut().take(want) { + *p = Some(self.samples(r.bytes(rowb * h)?, w, h)); + } + } + 1 => { + let mut counts = Vec::with_capacity(nch * h); + for _ in 0..nch * h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut out = Vec::with_capacity(rowb * h); + for row in 0..h { + unpack_bits(r.bytes(counts[c * h + row])?, rowb, &mut out)?; + } + *p = Some(self.samples(&out, w, h)); + } + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut plane = v.get(c * rowb * h..(c + 1) * rowb * h).ok_or(Error::Truncated)?.to_vec(); + if comp == 3 { + self.unpredict(&mut plane, w, h); + } + *p = Some(self.samples(&plane, w, h)); + } + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + } + let alpha = if self.merged_alpha && nch > cc { planes[cc].take() } else { None }; + let mut data = Vec::with_capacity(w * h); + for i in 0..w * h { + let [r, g, b] = self.to_rgb(&planes, i); + let a = alpha.as_ref().map_or(1.0, |a| a[i]); + data.push([r, g, b, a]); + } + Ok(Pixels { width: w as u32, height: h as u32, data }) + } + + /// One layer's pixels with its layer mask (and vector-less fill colour) applied to alpha. + /// `canvas`: document-sized with the layer in place; otherwise the layer's own bounds. + pub fn layer_pixels(&self, index: usize, canvas: bool) -> Result { + let l = self.layers.get(index).ok_or_else(|| Error::Invalid(format!("no layer {index}")))?; + let doc = Rect { top: 0, left: 0, bottom: self.height as i32, right: self.width as i32 }; + // Fill layers often have no pixel data: they cover the canvas (or the mask). + let fill_rect = + if l.fill.is_some() && !l.has_pixels() { l.mask.filter(|m| !m.rect.is_empty() && m.default_color == 0).map_or(doc, |m| m.rect) } else { l.rect }; + let out_rect = if canvas { + doc + } else if fill_rect.is_empty() { + Rect { top: 0, left: 0, bottom: 1, right: 1 } + } else { + fill_rect + }; + let (ow, oh) = (out_rect.width() as usize, out_rect.height() as usize); + let mut out = vec![[0.0f32; 4]; ow * oh]; + let src = fill_rect; + let (sw, sh) = (src.width() as usize, src.height() as usize); + if sw > 0 && sh > 0 { + let mut planes: Vec>> = vec![None; self.mode.color_channels()]; + let mut alpha = None; + if l.has_pixels() { + for c in &l.channels { + match c.id { + -1 => alpha = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + id if id >= 0 && (id as usize) < planes.len() => planes[id as usize] = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + _ => {} + } + } + } + let fill = l.fill.map(|c| c.map(|v| v as f32)); + for y in 0..sh { + let dy = src.top + y as i32 - out_rect.top; + if dy < 0 || dy >= oh as i32 { + continue; + } + for x in 0..sw { + let dx = src.left + x as i32 - out_rect.left; + if dx < 0 || dx >= ow as i32 { + continue; + } + let i = y * sw + x; + let (rgb, a) = match fill { + Some(f) if !l.has_pixels() => (f, 1.0), + _ => (self.to_rgb(&planes, i), alpha.as_ref().map_or(1.0, |a| a[i])), + }; + out[dy as usize * ow + dx as usize] = [rgb[0], rgb[1], rgb[2], a]; + } + } + } + // Layer mask: multiplies alpha (also outside the mask rect, by its default colour). + if let Some(m) = l.mask.filter(|m| !m.disabled) { + let mc = l.channels.iter().find(|c| c.id == -2); + let (mw, mh) = (m.rect.width() as usize, m.rect.height() as usize); + let mask = match mc { + Some(c) if mw > 0 && mh > 0 => Some(self.decode_channel(c.offset, c.len, mw, mh)?), + _ => None, + }; + let def = m.default_color as f32 / 255.0; + for y in 0..oh { + let py = out_rect.top + y as i32; + for x in 0..ow { + let px = out_rect.left + x as i32; + let inside = py >= m.rect.top && py < m.rect.bottom && px >= m.rect.left && px < m.rect.right; + let mut v = match (&mask, inside) { + (Some(mk), true) => mk[(py - m.rect.top) as usize * mw + (px - m.rect.left) as usize], + _ => def, + }; + if m.inverted { + v = 1.0 - v; + } + out[y * ow + x][3] *= v; + } + } + } + Ok(Pixels { width: ow as u32, height: oh as u32, data: out }) + } +} + +/// The next additional-information block at `*pos` (signature `8BIM`/`8B64`, key, length): +/// returns (key, data start, data end). Skips up to 3 padding bytes before a signature. +fn next_block(data: &[u8], pos: &mut usize, end: usize, psb: bool) -> Option<([u8; 4], usize, usize)> { + let mut p = *pos; + for _ in 0..4 { + if p + 12 > end { + return None; + } + if &data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64" { + break; + } + p += 1; + } + if p + 12 > end || !(&data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64") { + return None; + } + let mut key = [0u8; 4]; + key.copy_from_slice(&data[p + 4..p + 8]); + let wide = + psb && matches!(&key, b"LMsk" | b"Lr16" | b"Lr32" | b"Layr" | b"Mt16" | b"Mt32" | b"Mtrn" | b"Alph" | b"FMsk" | b"lnk2" | b"FEid" | b"FXid" | b"PxSD"); + let mut r = Reader::at(&data[..end], p + 8); + let len = r.length(wide).ok()?; + let start = r.pos; + let stop = start.checked_add(len)?; + if stop > end { + return None; + } + *pos = stop; + Some((key, start, stop)) +} + +/// PackBits decoding of one row (appends exactly `row` bytes, zero-padded). +fn unpack_bits(src: &[u8], row: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let mut i = 0; + while i < src.len() && out.len() - start < row { + let n = src[i] as i8; + i += 1; + if n >= 0 { + let k = n as usize + 1; + let s = src.get(i..i + k).ok_or(Error::Truncated)?; + out.extend_from_slice(s); + i += k; + } else if n != -128 { + let k = (1 - n as isize) as usize; + let v = *src.get(i).ok_or(Error::Truncated)?; + out.extend(std::iter::repeat_n(v, k)); + i += 1; + } + } + out.resize(start + row, 0); + Ok(()) +} + +/// PackBits encoding of one row. +pub(crate) fn pack_bits(src: &[u8]) -> Vec { + let mut out = vec![]; + let mut i = 0; + while i < src.len() { + // run of identical bytes + let mut run = 1; + while i + run < src.len() && src[i + run] == src[i] && run < 128 { + run += 1; + } + if run >= 2 { + out.push((1 - run as isize) as i8 as u8); + out.push(src[i]); + i += run; + continue; + } + // literal run + let start = i; + let mut n = 0; + while i < src.len() && n < 128 { + if i + 1 < src.len() && src[i + 1] == src[i] { + break; + } + i += 1; + n += 1; + } + if n == 0 { + i += 1; + n = 1; + } + out.push((n - 1) as u8); + out.extend_from_slice(&src[start..start + n]); + } + out +} + +fn srgb_encode(v: f32) -> f32 { + if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } +} + +/// CIE L*a*b* (D50) to sRGB (D65 via the Bradford-adapted matrix), clamped. +fn lab_to_rgb(l: f32, a: f32, b: f32) -> [f32; 3] { + let fy = (l + 16.0) / 116.0; + let fx = fy + a / 500.0; + let fz = fy - b / 200.0; + let f = |t: f32| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f32 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + let (x, y, z) = (0.964_22 * f(fx), f(fy), 0.825_21 * f(fz)); + // XYZ (D50) → linear sRGB. + let r = 3.133_856 * x - 1.616_867 * y - 0.490_615 * z; + let g = -0.978_768 * x + 1.916_142 * y + 0.033_454 * z; + let bb = 0.071_945 * x - 0.228_991 * y + 1.405_243 * z; + [r, g, bb].map(|v| srgb_encode(v.clamp(0.0, 1.0))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/psd/src/read.rs b/crates/psd/src/read.rs new file mode 100644 index 000000000..97651b2ba --- /dev/null +++ b/crates/psd/src/read.rs @@ -0,0 +1,86 @@ +//! Big-endian byte reader. + +use crate::{Error, Result}; + +#[derive(Clone)] +pub struct Reader<'a> { + pub data: &'a [u8], + pub pos: usize, +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data, pos: 0 } + } + pub fn at(data: &'a [u8], pos: usize) -> Self { + Reader { data, pos } + } + pub fn remaining(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> { + if self.remaining() < n { + return Err(Error::Truncated); + } + let s = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + pub fn skip(&mut self, n: usize) -> Result<()> { + self.bytes(n).map(|_| ()) + } + pub fn u8(&mut self) -> Result { + Ok(self.bytes(1)?[0]) + } + pub fn u16(&mut self) -> Result { + let b = self.bytes(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + pub fn i16(&mut self) -> Result { + Ok(self.u16()? as i16) + } + pub fn u32(&mut self) -> Result { + let b = self.bytes(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + pub fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + pub fn u64(&mut self) -> Result { + let b = self.bytes(8)?; + let mut a = [0u8; 8]; + a.copy_from_slice(b); + Ok(u64::from_be_bytes(a)) + } + pub fn f32(&mut self) -> Result { + Ok(f32::from_bits(self.u32()?)) + } + pub fn f64(&mut self) -> Result { + Ok(f64::from_bits(self.u64()?)) + } + pub fn tag(&mut self) -> Result<[u8; 4]> { + let b = self.bytes(4)?; + Ok([b[0], b[1], b[2], b[3]]) + } + /// A length field: 4 bytes in PSD, 8 bytes in PSB (`wide`). + pub fn length(&mut self, wide: bool) -> Result { + let v = if wide { self.u64()? } else { self.u32()? as u64 }; + usize::try_from(v).map_err(|_| Error::Invalid("length".into())) + } + /// A Pascal string padded so that length byte + text is a multiple of `pad`. + pub fn pascal(&mut self, pad: usize) -> Result { + let n = self.u8()? as usize; + let s = self.bytes(n)?; + let total = 1 + n; + let padded = total.div_ceil(pad) * pad; + self.skip(padded - total)?; + // Pascal strings are in the system (Mac Roman) encoding; ASCII covers the common case. + Ok(s.iter().map(|&b| if b < 0x80 { b as char } else { mac_roman(b) }).collect()) + } +} + +/// Mac OS Roman upper half (0x80..=0xFF), as published by the Unicode Consortium mapping table. +fn mac_roman(b: u8) -> char { + const T: &str = "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ"; + T.chars().nth((b - 0x80) as usize).unwrap_or('?') +} diff --git a/crates/psd/src/tests.rs b/crates/psd/src/tests.rs new file mode 100644 index 000000000..68680fe71 --- /dev/null +++ b/crates/psd/src/tests.rs @@ -0,0 +1,189 @@ +use super::write::*; +use super::*; + +fn gradient(w: u32, h: u32) -> Vec<[f32; 4]> { + (0..w * h).map(|i| [(i % w) as f32 / (w - 1).max(1) as f32, (i / w) as f32 / (h - 1).max(1) as f32, 0.25, 1.0]).collect() +} + +fn close(a: &[[f32; 4]], b: &[[f32; 4]], tol: f32) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (0..4).all(|c| (x[c] - y[c]).abs() <= tol)) +} + +fn doc_with_layers(depth: u16, mode: ColorMode, rle: bool) -> WDoc { + let mut d = WDoc::new(40, 30); + d.depth = depth; + d.mode = mode; + d.rle = rle; + let mut bg = WLayer::pixels("Background", Rect::new(0, 0, 40, 30), gradient(40, 30)); + bg.opacity = 255; + let mut red = WLayer::solid("Red box", Rect::new(5, 6, 10, 8), [1.0, 0.0, 0.0, 1.0]).with_blend(b"mul ").with_opacity(128); + red.hidden = true; + let mut masked = WLayer::solid("Masked", Rect::new(20, 10, 12, 12), [0.0, 0.5, 1.0, 1.0]).with_blend(b"scrn"); + // Mask: left half opaque, right half transparent; default (outside) black. + let mrect = Rect::new(20, 10, 12, 12); + masked.mask = Some(WMask { rect: mrect, data: (0..144).map(|i| if i % 12 < 6 { 1.0 } else { 0.0 }).collect(), default_color: 0, disabled: false }); + d.layers = vec![bg, red, masked]; + d +} + +#[test] +fn layers_round_trip_8bit_rle() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + assert!(is_psd(&bytes)); + let p = Psd::parse(bytes).unwrap(); + assert_eq!((p.width, p.height, p.depth, p.mode), (40, 30, 8, ColorMode::Rgb)); + assert_eq!(p.layers.len(), 3); + let names: Vec<&str> = p.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Background", "Red box", "Masked"]); + assert_eq!(p.layers[1].blend_key(), "mul "); + assert_eq!(p.layers[1].opacity, 128); + assert!(p.layers[1].hidden && !p.layers[0].hidden); + assert_eq!(p.layers[2].blend_key(), "scrn"); + assert_eq!(p.layers[1].rect, Rect::new(5, 6, 10, 8)); + // Pixels (8-bit quantisation). + let bg = p.layer_pixels(0, false).unwrap(); + assert!(close(&bg.data, &gradient(40, 30), 1.0 / 255.0)); + // Retain layer size vs canvas. + let red = p.layer_pixels(1, false).unwrap(); + assert_eq!((red.width, red.height), (10, 8)); + let red_c = p.layer_pixels(1, true).unwrap(); + assert_eq!((red_c.width, red_c.height), (40, 30)); + assert_eq!(red_c.data[6 * 40 + 5], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(red_c.data[0][3], 0.0); + // Layer mask becomes alpha. + let m = p.layer_pixels(2, false).unwrap(); + assert_eq!(m.data[0][3], 1.0); + assert_eq!(m.data[11][3], 0.0); + // Merged composite. + let c = p.composite().unwrap(); + assert_eq!((c.width, c.height), (40, 30)); + assert!(p.merged_alpha); +} + +#[test] +fn layers_round_trip_16bit_raw_and_cmyk_gray() { + for (depth, mode, rle) in + [(16, ColorMode::Rgb, false), (16, ColorMode::Rgb, true), (8, ColorMode::Cmyk, true), (16, ColorMode::Cmyk, false), (8, ColorMode::Grayscale, false)] + { + let bytes = write(&doc_with_layers(depth, mode, rle)); + let p = Psd::parse(bytes).unwrap_or_else(|e| panic!("{depth} {mode:?} {rle}: {e}")); + assert_eq!(p.layers.len(), 3, "{depth} {mode:?}"); + assert_eq!(p.depth, depth); + let bg = p.layer_pixels(0, true).unwrap(); + let tol = if depth == 16 { 1.0 / 30000.0 } else { 1.5 / 255.0 }; + let want: Vec<[f32; 4]> = + if mode == ColorMode::Grayscale { gradient(40, 30).iter().map(|p| [p[0], p[0], p[0], 1.0]).collect() } else { gradient(40, 30) }; + assert!(close(&bg.data, &want, tol), "{depth} {mode:?} {rle}: {:?} vs {:?}", bg.data[41], want[41]); + let c = p.composite().unwrap(); + assert_eq!(c.data.len(), 1200); + } +} + +#[test] +fn groups_build_a_tree() { + let mut d = WDoc::new(16, 16); + d.layers = vec![ + WLayer::solid("Bottom", Rect::new(0, 0, 16, 16), [1.0; 4]), + WLayer::group_end(), + WLayer::solid("Inner A", Rect::new(0, 0, 4, 4), [1.0, 0.0, 0.0, 1.0]), + WLayer::group_end(), + WLayer::solid("Deep", Rect::new(2, 2, 4, 4), [0.0, 1.0, 0.0, 1.0]), + WLayer::group("Nested", false, *b"norm"), + WLayer::group("Group 1", true, *b"pass"), + WLayer::solid("Top", Rect::new(8, 8, 4, 4), [0.0, 0.0, 1.0, 1.0]), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[6].section, Section::OpenFolder); + assert_eq!(p.layers[6].blend_key(), "pass"); + assert_eq!(p.layers[5].section, Section::ClosedFolder); + let t = p.tree(); + assert_eq!( + t, + vec![ + Node::Layer(7), + Node::Group { layer: 6, children: vec![Node::Group { layer: 5, children: vec![Node::Layer(4)] }, Node::Layer(2)] }, + Node::Layer(0) + ] + ); +} + +#[test] +fn text_effects_fill_and_adjustments() { + let mut d = WDoc::new(64, 32); + let fx = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 1.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 7.0)), + ), + ); + d.layers = vec![ + WLayer::empty("Fill", vec![solid_color_block([0.0, 1.0, 0.0])]), + WLayer::empty("Title", vec![text_block("Hello\nWorld", [10.0, 20.0], "Inter-Bold", 18.0, [1.0, 0.0, 0.0], 2)]).with_block(effects_block(&fx)), + WLayer::empty("Hue", vec![hue_sat_block(30, -20, 10)]), + WLayer::empty("Lev", vec![levels_block(10, 240, 5, 250, 1.2)]), + WLayer::empty("Inv", vec![invert_block()]), + WLayer::empty("BC", vec![brightness_block(20, -10)]), + WLayer::empty("Exp", vec![exposure_block(1.5, 0.0, 1.0)]), + WLayer::solid("Vec", Rect::new(0, 0, 64, 32), [1.0; 4]).with_block(vector_mask_block( + &[vec![[[8.0, 4.0]; 3], [[40.0, 4.0]; 3], [[40.0, 20.0]; 3]]], + 64, + 32, + false, + )), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[0].fill, Some([0.0, 1.0, 0.0])); + let fill = p.layer_pixels(0, false).unwrap(); + assert_eq!((fill.width, fill.height), (64, 32)); + assert_eq!(fill.data[5], [0.0, 1.0, 0.0, 1.0]); + let t = p.layers[1].text.as_ref().unwrap(); + assert_eq!(t.text, "Hello\nWorld"); + assert_eq!(t.transform[4..], [10.0, 20.0]); + assert_eq!(t.style.font.as_deref(), Some("Inter-Bold")); + assert_eq!(t.style.size, Some(18.0)); + assert_eq!(t.style.color, Some([1.0, 0.0, 0.0])); + assert_eq!(t.style.justification, Some(2)); + let e = p.layers[1].effects.as_ref().unwrap(); + let ds = e.obj("DrSh").unwrap(); + assert_eq!(ds.enum_value("Md "), Some("Mltp")); + assert_eq!(ds.num("Dstn"), Some(7.0)); + assert_eq!(p.layers[2].adjustment, Some(Adjustment::HueSaturation { hue: 30.0, saturation: -20.0, lightness: 10.0, colorize: false })); + assert_eq!(p.layers[3].adjustment, Some(Adjustment::Levels { in_black: 10.0, in_white: 240.0, gamma: 1.2, out_black: 5.0, out_white: 250.0 })); + assert_eq!(p.layers[4].adjustment, Some(Adjustment::Invert)); + assert_eq!(p.layers[5].adjustment, Some(Adjustment::BrightnessContrast { brightness: 20.0, contrast: -10.0, legacy: true })); + assert_eq!(p.layers[6].adjustment, Some(Adjustment::Exposure { exposure: 1.5, offset: 0.0, gamma: 1.0 })); + let vm = p.layers[7].vector_mask.as_ref().unwrap(); + assert_eq!(vm.subpaths.len(), 1); + assert!(vm.subpaths[0].closed); + let k = &vm.subpaths[0].knots; + assert_eq!(k.len(), 3); + assert!((k[1][1][0] - 40.0).abs() < 1e-4 && (k[2][1][1] - 20.0).abs() < 1e-4); +} + +#[test] +fn packbits_round_trip() { + for src in [vec![], vec![1u8], vec![5; 300], (0..=255).collect::>(), vec![1, 1, 2, 3, 3, 3, 4, 5, 5]] { + let enc = pack_bits(&src); + let mut out = vec![]; + unpack_bits(&enc, src.len(), &mut out).unwrap(); + assert_eq!(out, src); + } +} + +#[test] +fn truncated_and_garbage_input_errors() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + for n in [0, 3, 10, 26, 40, 100, bytes.len() / 2] { + let r = Psd::parse(bytes[..n].to_vec()); + if let Ok(p) = r { + let _ = p.composite(); + } + } + assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err()); + assert!(Psd::parse(b"PNG".to_vec()).is_err()); +} diff --git a/crates/psd/src/write.rs b/crates/psd/src/write.rs new file mode 100644 index 000000000..fb78cc78e --- /dev/null +++ b/crates/psd/src/write.rs @@ -0,0 +1,491 @@ +//! A minimal Photoshop document writer, per the same specification as the reader: RGB, CMYK or +//! Grayscale, 8 or 16 bits, raw or RLE channels, pixel layers with layer masks, groups and any +//! additional-information blocks (helpers below build text, effects, fill and adjustment +//! blocks). Used to generate test fixtures; not a full exporter. + +use crate::descriptor::{self, DValue, Descriptor}; +use crate::{ColorMode, Rect, Section, engine_data, pack_bits}; + +/// A layer mask for [`WLayer`]. +#[derive(Clone, Debug)] +pub struct WMask { + pub rect: Rect, + /// `rect.width() * rect.height()` values in 0..1. + pub data: Vec, + pub default_color: u8, + pub disabled: bool, +} + +/// A layer to write. +#[derive(Clone, Debug)] +pub struct WLayer { + pub name: String, + pub rect: Rect, + /// Straight RGBA, `rect.width() * rect.height()` pixels. + pub pixels: Vec<[f32; 4]>, + pub blend: [u8; 4], + pub opacity: u8, + pub hidden: bool, + pub clipping: bool, + pub mask: Option, + pub section: Section, + /// Extra additional-information blocks (key, data). + pub blocks: Vec<([u8; 4], Vec)>, +} + +impl WLayer { + /// A pixel layer. + pub fn pixels(name: &str, rect: Rect, pixels: Vec<[f32; 4]>) -> WLayer { + WLayer { + name: name.into(), + rect, + pixels, + blend: *b"norm", + opacity: 255, + hidden: false, + clipping: false, + mask: None, + section: Section::Layer, + blocks: vec![], + } + } + /// A layer filled with one colour. + pub fn solid(name: &str, rect: Rect, rgba: [f32; 4]) -> WLayer { + let n = (rect.width() * rect.height()) as usize; + WLayer::pixels(name, rect, vec![rgba; n]) + } + /// The top record of a group (its children come *before* it in file order, after a + /// [`WLayer::group_end`]). + pub fn group(name: &str, open: bool, blend: [u8; 4]) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.section = if open { Section::OpenFolder } else { Section::ClosedFolder }; + l.blend = blend; + l + } + /// The hidden record that closes a group (written below its children). + pub fn group_end() -> WLayer { + let mut l = WLayer::pixels("", Rect::default(), vec![]); + l.section = Section::Divider; + l + } + /// A layer without pixels carrying `blocks` (adjustment and fill layers, type layers). + pub fn empty(name: &str, blocks: Vec<([u8; 4], Vec)>) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.blocks = blocks; + l + } + pub fn with_blend(mut self, key: &[u8; 4]) -> WLayer { + self.blend = *key; + self + } + pub fn with_opacity(mut self, o: u8) -> WLayer { + self.opacity = o; + self + } + pub fn with_block(mut self, b: ([u8; 4], Vec)) -> WLayer { + self.blocks.push(b); + self + } +} + +/// A document to write. +#[derive(Clone, Debug)] +pub struct WDoc { + pub width: u32, + pub height: u32, + /// 8 or 16. + pub depth: u16, + /// Rgb, Cmyk or Grayscale. + pub mode: ColorMode, + pub rle: bool, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers + /// with Normal blending. + pub composite: Option>, +} + +impl WDoc { + pub fn new(width: u32, height: u32) -> WDoc { + WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None } + } +} + +fn u16b(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} +fn u32b(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Colour channels for a straight RGB pixel in the document mode. +fn mode_channels(mode: ColorMode, p: [f32; 4]) -> Vec { + match mode { + ColorMode::Cmyk => { + // Stored values are inverted ink amounts: k = max, c = r / max (reader: r = c * k). + let m = p[0].max(p[1]).max(p[2]); + if m <= 0.0 { vec![0.0, 0.0, 0.0, 0.0] } else { vec![p[0] / m, p[1] / m, p[2] / m, m] } + } + ColorMode::Grayscale => vec![p[0]], + _ => vec![p[0], p[1], p[2]], + } +} + +fn encode_samples(v: &[f32], depth: u16) -> Vec { + let mut out = Vec::with_capacity(v.len() * if depth == 16 { 2 } else { 1 }); + for &x in v { + let x = x.clamp(0.0, 1.0); + if depth == 16 { + u16b(&mut out, (x * 65535.0).round() as u16); + } else { + out.push((x * 255.0).round() as u8); + } + } + out +} + +/// One channel's data (compression field + data) for a `w`×`h` plane. +fn channel_data(v: &[f32], w: usize, h: usize, depth: u16, rle: bool) -> Vec { + let raw = encode_samples(v, depth); + let mut out = vec![]; + let rowb = raw.len() / h.max(1); + if !rle || w == 0 || h == 0 { + u16b(&mut out, 0); + out.extend_from_slice(&raw); + return out; + } + u16b(&mut out, 1); + let rows: Vec> = raw.chunks(rowb.max(1)).map(pack_bits).collect(); + for r in &rows { + u16b(&mut out, r.len() as u16); + } + for r in rows { + out.extend_from_slice(&r); + } + out +} + +fn pascal4(out: &mut Vec, s: &str) { + let b: Vec = s.bytes().map(|c| if c < 0x80 { c } else { b'?' }).take(255).collect(); + out.push(b.len() as u8); + out.extend_from_slice(&b); + let total = 1 + b.len(); + out.resize(out.len() + total.div_ceil(4) * 4 - total, 0); +} + +fn block(out: &mut Vec, key: &[u8; 4], data: &[u8]) { + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(key); + let padded = data.len().div_ceil(2) * 2; + u32b(out, padded as u32); + out.extend_from_slice(data); + out.resize(out.len() + padded - data.len(), 0); +} + +fn layer_info(doc: &WDoc) -> Vec { + let mut out = vec![]; + out.extend_from_slice(&(-(doc.layers.len() as i16)).to_be_bytes()); + let mut chan_data: Vec)>> = vec![]; + for l in &doc.layers { + let (w, h) = (l.rect.width() as usize, l.rect.height() as usize); + let mut chans: Vec<(i16, Vec)> = vec![]; + let n = w * h; + let alpha: Vec = l.pixels.iter().take(n).map(|p| p[3]).collect(); + chans.push((-1, channel_data(&alpha, w, h, doc.depth, doc.rle))); + let cc = doc.mode.color_channels(); + let planes: Vec> = l.pixels.iter().take(n).map(|p| mode_channels(doc.mode, *p)).collect(); + for c in 0..cc { + let plane: Vec = planes.iter().map(|v| v[c]).collect(); + chans.push((c as i16, channel_data(&plane, w, h, doc.depth, doc.rle))); + } + if let Some(m) = &l.mask { + chans.push((-2, channel_data(&m.data, m.rect.width() as usize, m.rect.height() as usize, doc.depth, doc.rle))); + } + // record + for v in [l.rect.top, l.rect.left, l.rect.bottom, l.rect.right] { + out.extend_from_slice(&v.to_be_bytes()); + } + u16b(&mut out, chans.len() as u16); + for (id, d) in &chans { + out.extend_from_slice(&id.to_be_bytes()); + u32b(&mut out, d.len() as u32); + } + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(&l.blend); + out.push(l.opacity); + out.push(l.clipping as u8); + let mut flags = 8u8; + if l.hidden { + flags |= 2; + } + out.push(flags); + out.push(0); + let mut extra = vec![]; + match &l.mask { + Some(m) => { + u32b(&mut extra, 20); + for v in [m.rect.top, m.rect.left, m.rect.bottom, m.rect.right] { + extra.extend_from_slice(&v.to_be_bytes()); + } + extra.push(m.default_color); + extra.push(if m.disabled { 2 } else { 0 }); + extra.extend_from_slice(&[0, 0]); + } + None => u32b(&mut extra, 0), + } + u32b(&mut extra, 0); // blending ranges + pascal4(&mut extra, &l.name); + let luni = unicode_plain(&l.name); + block(&mut extra, b"luni", &luni); + match l.section { + Section::Layer => {} + s => { + let mut d = vec![]; + u32b( + &mut d, + match s { + Section::OpenFolder => 1, + Section::ClosedFolder => 2, + _ => 3, + }, + ); + d.extend_from_slice(b"8BIM"); + d.extend_from_slice(&l.blend); + block(&mut extra, b"lsct", &d); + } + } + for (k, d) in &l.blocks { + block(&mut extra, k, d); + } + u32b(&mut out, extra.len() as u32); + out.extend_from_slice(&extra); + chan_data.push(chans); + } + for chans in chan_data { + for (_, d) in chans { + out.extend_from_slice(&d); + } + } + if out.len() % 2 == 1 { + out.push(0); + } + out +} + +fn unicode_plain(s: &str) -> Vec { + let v: Vec = s.encode_utf16().collect(); + let mut out = vec![]; + u32b(&mut out, v.len() as u32); + for c in v { + u16b(&mut out, c); + } + out +} + +/// Normal "over" of the visible pixel layers (groups ignored), straight alpha. +fn flatten(doc: &WDoc) -> Vec<[f32; 4]> { + let (w, h) = (doc.width as i32, doc.height as i32); + let mut acc = vec![[0.0f32; 4]; (w * h) as usize]; + for l in doc.layers.iter().filter(|l| !l.hidden && l.section == Section::Layer) { + let lw = l.rect.width() as i32; + for (i, p) in l.pixels.iter().enumerate() { + let (x, y) = (l.rect.left + i as i32 % lw.max(1), l.rect.top + i as i32 / lw.max(1)); + if x < 0 || y < 0 || x >= w || y >= h { + continue; + } + let a = p[3] * l.opacity as f32 / 255.0; + let d = &mut acc[(y * w + x) as usize]; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (p[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +/// Serialise a document. +pub fn write(doc: &WDoc) -> Vec { + let cc = doc.mode.color_channels(); + let mut out = vec![]; + out.extend_from_slice(b"8BPS"); + u16b(&mut out, 1); + out.extend_from_slice(&[0; 6]); + u16b(&mut out, (cc + 1) as u16); + u32b(&mut out, doc.height); + u32b(&mut out, doc.width); + u16b(&mut out, doc.depth); + u16b(&mut out, doc.mode.to_u16()); + u32b(&mut out, 0); // colour mode data + u32b(&mut out, 0); // image resources + // Layer and mask information. + let mut lm = vec![]; + if !doc.layers.is_empty() { + let li = layer_info(doc); + if doc.depth == 16 { + u32b(&mut lm, 0); + u32b(&mut lm, 0); // global layer mask info + let mut padded = li; + while !padded.len().is_multiple_of(4) { + padded.push(0); + } + lm.extend_from_slice(b"8BIM"); + lm.extend_from_slice(b"Lr16"); + u32b(&mut lm, padded.len() as u32); + lm.extend_from_slice(&padded); + } else { + u32b(&mut lm, li.len() as u32); + lm.extend_from_slice(&li); + u32b(&mut lm, 0); + } + } + u32b(&mut out, lm.len() as u32); + out.extend_from_slice(&lm); + // Merged image. + let comp = doc.composite.clone().unwrap_or_else(|| flatten(doc)); + let (w, h) = (doc.width as usize, doc.height as usize); + let planes: Vec> = comp.iter().map(|p| mode_channels(doc.mode, *p)).collect(); + let mut chans: Vec> = (0..cc).map(|c| planes.iter().map(|v| v[c]).collect()).collect(); + chans.push(comp.iter().map(|p| p[3]).collect()); + if doc.rle { + u16b(&mut out, 1); + let rows: Vec>> = + chans.iter().map(|c| encode_samples(c, doc.depth).chunks((w * if doc.depth == 16 { 2 } else { 1 }).max(1)).map(pack_bits).collect()).collect(); + for c in &rows { + for r in c { + u16b(&mut out, r.len() as u16); + } + } + for c in rows { + for r in c { + out.extend_from_slice(&r); + } + } + } else { + u16b(&mut out, 0); + for c in &chans { + out.extend_from_slice(&encode_samples(c, doc.depth)); + } + } + let _ = h; + out +} + +// ---------------------------------------------------------------- block helpers + +/// `TySh`: a type layer with one style run; the text origin sits at `origin`. +pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [1.0, 0.0, 0.0, 1.0, origin[0], origin[1]] { + d.extend_from_slice(&f64::to_be_bytes(v)); + } + u16b(&mut d, 50); + u32b(&mut d, 16); + let desc = Descriptor::new("TxLr") + .with("Txt ", DValue::Text(text.replace('\n', "\r"))) + .with("textGridding", DValue::Enum("textGridding".into(), "None".into())) + .with("Ornt", DValue::Enum("Ornt".into(), "Hrzn".into())) + .with("EngineData", DValue::Raw(engine_data::minimal(text, font, size, color, justification))); + descriptor::write_descriptor(&mut d, &desc); + // Warp: version, descriptor version, an empty warp descriptor. + u16b(&mut d, 1); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("warp").with("warpStyle", DValue::Enum("warpStyle".into(), "warpNone".into()))); + for _ in 0..4 { + u32b(&mut d, 0); + } + (*b"TySh", d) +} + +/// `lfx2`: layer effects from an effects descriptor (`null` class with `DrSh`, `FrFX`…). +pub fn effects_block(desc: &Descriptor) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 0); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, desc); + (*b"lfx2", d) +} + +/// `SoCo`: solid colour fill. +pub fn solid_color_block(rgb: [f64; 3]) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("null").with("Clr ", DValue::Descriptor(Descriptor::rgb(rgb)))); + (*b"SoCo", d) +} + +/// `hue2`: Hue/Saturation (master). +pub fn hue_sat_block(hue: i16, sat: i16, light: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + d.extend_from_slice(&[0, 0]); + for v in [0i16, 0, 0, hue, sat, light] { + d.extend_from_slice(&v.to_be_bytes()); + } + // Six colour ranges (unused). + d.resize(d.len() + 6 * 24, 0); + (*b"hue2", d) +} + +/// `levl`: Levels (composite record). +pub fn levels_block(in_black: i16, in_white: i16, out_black: i16, out_white: i16, gamma: f64) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + for v in [in_black, in_white, out_black, out_white, (gamma * 100.0).round() as i16] { + d.extend_from_slice(&v.to_be_bytes()); + } + for _ in 0..28 { + for v in [0i16, 255, 0, 255, 100] { + d.extend_from_slice(&v.to_be_bytes()); + } + } + (*b"levl", d) +} + +pub fn invert_block() -> ([u8; 4], Vec) { + (*b"nvrt", vec![]) +} + +pub fn brightness_block(brightness: i16, contrast: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + for v in [brightness, contrast, 127] { + d.extend_from_slice(&v.to_be_bytes()); + } + d.push(0); + (*b"brit", d) +} + +pub fn exposure_block(exposure: f32, offset: f32, gamma: f32) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [exposure, offset, gamma] { + d.extend_from_slice(&v.to_be_bytes()); + } + (*b"expA", d) +} + +/// `vmsk`: a vector mask from closed polygons/curves given as (in, anchor, out) knots in pixels. +pub fn vector_mask_block(subpaths: &[Vec<[[f64; 2]; 3]>], width: u32, height: u32, inverted: bool) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 3); + u32b(&mut d, inverted as u32); + // Path fill rule record. + u16b(&mut d, 6); + d.resize(d.len() + 24, 0); + let fx = |v: f64| ((v * (1 << 24) as f64).round() as i32).to_be_bytes(); + for sp in subpaths { + u16b(&mut d, 0); + u16b(&mut d, sp.len() as u16); + d.resize(d.len() + 22, 0); + for k in sp { + u16b(&mut d, 1); + for p in k { + d.extend_from_slice(&fx(p[1] / height as f64)); + d.extend_from_slice(&fx(p[0] / width as f64)); + } + } + } + (*b"vmsk", d) +} diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 80e304558..db2fa9dd0 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +lz4_flex = { version = "0.11", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] } effectcraft-model = { workspace = true } kurbo = { workspace = true } rayon = { workspace = true } diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index ca850a866..ac7993882 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -25,9 +25,13 @@ use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup}; use crate::eval::EvalCtx; -/// Thread-safe, memory-budgeted cache of processed layer buffers. +/// Thread-safe, memory-budgeted cache of processed layer buffers, optionally backed by the +/// persistent [`DiskCache`](crate::disk_cache::DiskCache): a memory miss reads the disk, and +/// buffers that were slow to render are written there (keys salted per project, see +/// [`LayerCache::set_disk`]). pub struct LayerCache { inner: Mutex, + disk: Mutex, u64)>>, } struct Entry { @@ -43,6 +47,8 @@ struct Inner { clock: u64, hits: u64, misses: u64, + /// When each recent miss happened (to time the render that follows it). + missed_at: HashMap, } /// Hit/miss counters and memory use (for perf readouts and tests). @@ -63,7 +69,37 @@ impl Default for LayerCache { impl LayerCache { /// A cache holding at most `budget` bytes of pixels. pub fn new(budget: usize) -> LayerCache { - LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0 }) } + LayerCache { + inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }), + disk: Mutex::new(None), + } + } + + /// Attach (or detach) the disk cache. `salt` separates projects and footage versions (see + /// `footage_salt`); keys on disk are `(salt, key)`. + pub fn set_disk(&self, disk: Option<(Arc, u64)>) { + if let Ok(mut d) = self.disk.lock() { + *d = disk; + } + } + + /// Change the disk salt (after footage changed) keeping the attached disk cache. + pub fn set_disk_salt(&self, salt: u64) { + if let Ok(mut d) = self.disk.lock() + && let Some((_, s)) = d.as_mut() + { + *s = salt; + } + } + + pub fn disk(&self) -> Option> { + self.disk.lock().ok()?.as_ref().map(|d| d.0.clone()) + } + + fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { + let d = self.disk.lock().ok()?; + let (dc, salt) = d.as_ref()?; + Some((dc.clone(), ((*salt as u128) << 64) | key as u128)) } pub fn get(&self, key: u64) -> Option> { @@ -79,12 +115,32 @@ impl LayerCache { } None => { g.misses += 1; - None + if g.missed_at.len() > 4096 { + g.missed_at.clear(); + } + g.missed_at.insert(key, web_time::Instant::now()); + drop(g); + // Disk cache: a hit comes back into memory. + let (dc, dk) = self.disk_key(key)?; + let buf = Arc::new(dc.get_layer(dk)?); + self.insert_mem(key, buf.clone()); + Some(buf) } } } pub fn insert(&self, key: u64, buf: Arc) { + let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key)); + if let Some((dc, dk)) = self.disk_key(key) { + let slow = started.is_some_and(|t| t.elapsed().as_secs_f64() * 1e3 >= dc.min_layer_ms() as f64); + if slow { + dc.put_layer(dk, &buf); + } + } + self.insert_mem(key, buf); + } + + fn insert_mem(&self, key: u64, buf: Arc) { let bytes = buf.img.data.len() * std::mem::size_of::(); let Ok(mut g) = self.inner.lock() else { return }; if bytes > g.budget / 4 { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs new file mode 100644 index 000000000..50c3d1d73 --- /dev/null +++ b/crates/render/src/disk_cache.rs @@ -0,0 +1,831 @@ +//! The persistent disk cache (Settings ▸ Media & Disk Cache ▸ Disk Cache): RAM-preview frames +//! and layer cache entries kept on disk between sessions, like After Effects' disk cache. +//! +//! * Entries are files keyed by a 128-bit content hash (`/v1//<2 hex>/<32 hex>.ecc`): +//! frames by the hash of the composition's content (the comp and everything it uses, footage +//! files' size and modification time included) plus frame, scale and view; layer buffers by +//! the layer cache's content key salted with the project's footage fingerprint. A changed +//! project therefore never reads stale pixels: its keys simply change. +//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer +//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh +//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! are deleted. +//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the +//! folder when the cache opens (file modification times order it), so the cache survives +//! restarts; reads refresh a file's time. +//! * Safe concurrent access: writes go to a unique temporary file that is renamed into place +//! (atomic on every desktop OS), so readers in this or another process never see a partial +//! entry; a missing or damaged file is just a miss. Writes run on a background thread with a +//! bounded queue (a full queue drops the write rather than stalling rendering). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use effectcraft_effects::Buf; +use effectcraft_raster::Image; + +/// File format version (part of the folder layout). +const VERSION: &str = "v1"; +const MAGIC: &[u8; 4] = b"ECDC"; +/// Pending writes before new ones are dropped. +#[cfg(not(target_arch = "wasm32"))] +const QUEUE_LIMIT: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Kind { + Frame, + Layer, +} + +impl Kind { + fn dir(self) -> &'static str { + match self { + Kind::Frame => "frames", + Kind::Layer => "layers", + } + } + fn tag(self) -> u8 { + match self { + Kind::Frame => 1, + Kind::Layer => 2, + } + } +} + +/// A cached viewer frame: 8-bit premultiplied RGBA. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame8 { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} + +/// Counters for tests, the Info panel and `cache.diskStats`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiskStats { + pub entries: usize, + pub bytes: u64, + pub max_bytes: u64, + pub hits: u64, + pub misses: u64, + pub writes: u64, + pub evictions: u64, + pub dropped: u64, +} + +struct Meta { + bytes: u64, + last_use: u64, +} + +#[derive(Default)] +struct Index { + map: HashMap<(Kind, u128), Meta>, + total: u64, + clock: u64, +} + +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +enum Job { + Write(Kind, u128, Vec), +} + +struct Queue { + jobs: std::collections::VecDeque, + busy: bool, +} + +pub struct DiskCache { + dir: PathBuf, + index: Mutex, + max_bytes: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + writes: AtomicU64, + evictions: AtomicU64, + dropped: AtomicU64, + tmp_counter: AtomicU64, + queue: Arc<(Mutex, Condvar)>, + /// Layer buffers that took less than this to render are not written (ms). + min_layer_ms: AtomicU64, +} + +/// FNV-1a 128-bit (content keys). +#[derive(Clone, Copy)] +pub struct Hash128(u128); + +impl Default for Hash128 { + fn default() -> Self { + Hash128(0x6c62_272e_07bb_0142_62b8_2175_6295_c58d) + } +} + +impl Hash128 { + pub fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 ^= b as u128; + self.0 = self.0.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + } + pub fn write_u64(&mut self, v: u64) { + self.write(&v.to_le_bytes()); + } + pub fn finish(&self) -> u128 { + self.0 + } +} + +fn checksum(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// The platform's default disk cache folder. +pub fn default_folder() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from); + if cfg!(target_os = "macos") + && let Some(h) = &home + { + return h.join("Library/Caches/EffectCraft/Disk Cache"); + } + if cfg!(windows) + && let Some(l) = std::env::var_os("LOCALAPPDATA") + { + return PathBuf::from(l).join("EffectCraft").join("Disk Cache"); + } + if let Some(x) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(x).join("effectcraft/disk-cache"); + } + home.map(|h| h.join(".cache/effectcraft/disk-cache")).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-disk-cache")) +} + +impl DiskCache { + /// Open (or create) the cache in `folder`, holding at most `max_bytes`. + pub fn open(folder: impl AsRef, max_bytes: u64) -> std::io::Result> { + let dir = folder.as_ref().join(VERSION); + std::fs::create_dir_all(&dir)?; + let dc = Arc::new(DiskCache { + dir, + index: Mutex::new(Index::default()), + max_bytes: AtomicU64::new(max_bytes), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + writes: AtomicU64::new(0), + evictions: AtomicU64::new(0), + dropped: AtomicU64::new(0), + tmp_counter: AtomicU64::new(0), + queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: false }), Condvar::new())), + min_layer_ms: AtomicU64::new(30), + }); + dc.rescan(); + dc.evict(); + #[cfg(not(target_arch = "wasm32"))] + { + let weak = Arc::downgrade(&dc); + let q = dc.queue.clone(); + std::thread::Builder::new().name("ec-disk-cache".into()).spawn(move || writer(weak, q))?; + } + Ok(dc) + } + + pub fn folder(&self) -> &Path { + self.dir.parent().unwrap_or(&self.dir) + } + + /// Rebuild the index from the folder (oldest files first). + pub fn rescan(&self) { + let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; + for kind in [Kind::Frame, Kind::Layer] { + let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; + for shard in shards.flatten() { + let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; + for f in files.flatten() { + let p = f.path(); + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if name.contains(".tmp") { + // A writer that died mid-write; old temporaries are removed. + if f.metadata().and_then(|m| m.modified()).is_ok_and(|t| t.elapsed().is_ok_and(|e| e.as_secs() > 600)) { + let _ = std::fs::remove_file(&p); + } + continue; + } + let Some(hex) = name.strip_suffix(".ecc") else { continue }; + let Ok(key) = u128::from_str_radix(hex, 16) else { continue }; + let Ok(md) = f.metadata() else { continue }; + found.push((md.modified().unwrap_or(std::time::UNIX_EPOCH), kind, key, md.len())); + } + } + } + found.sort_by_key(|e| e.0); + let mut idx = Index::default(); + for (_, kind, key, bytes) in found { + idx.clock += 1; + let clock = idx.clock; + idx.total += bytes; + idx.map.insert((kind, key), Meta { bytes, last_use: clock }); + } + if let Ok(mut g) = self.index.lock() { + *g = idx; + } + } + + fn path(&self, kind: Kind, key: u128) -> PathBuf { + let hex = format!("{key:032x}"); + self.dir.join(kind.dir()).join(&hex[..2]).join(format!("{hex}.ecc")) + } + + pub fn max_bytes(&self) -> u64 { + self.max_bytes.load(Ordering::Relaxed) + } + + /// Change the size limit (evicts the least recently used entries when shrinking). + pub fn set_max_bytes(&self, b: u64) { + self.max_bytes.store(b, Ordering::Relaxed); + self.evict(); + } + + /// Layer buffers faster to render than this are not worth a disk write. + pub fn set_min_layer_ms(&self, ms: u64) { + self.min_layer_ms.store(ms, Ordering::Relaxed); + } + + pub fn min_layer_ms(&self) -> u64 { + self.min_layer_ms.load(Ordering::Relaxed) + } + + pub fn contains(&self, kind: Kind, key: u128) -> bool { + self.index.lock().is_ok_and(|g| g.map.contains_key(&(kind, key))) + } + + /// Keys of one kind (unordered). + pub fn keys(&self, kind: Kind) -> Vec { + self.index.lock().map(|g| g.map.keys().filter(|k| k.0 == kind).map(|k| k.1).collect()).unwrap_or_default() + } + + pub fn stats(&self) -> DiskStats { + let (entries, bytes) = self.index.lock().map(|g| (g.map.len(), g.total)).unwrap_or_default(); + DiskStats { + entries, + bytes, + max_bytes: self.max_bytes(), + hits: self.hits.load(Ordering::Relaxed), + misses: self.misses.load(Ordering::Relaxed), + writes: self.writes.load(Ordering::Relaxed), + evictions: self.evictions.load(Ordering::Relaxed), + dropped: self.dropped.load(Ordering::Relaxed), + } + } + + /// Delete every entry (Edit ▸ Purge ▸ All Disk Cache / Empty Disk Cache). + pub fn clear(&self) { + self.flush(); + if let Ok(mut g) = self.index.lock() { + *g = Index::default(); + } + for kind in [Kind::Frame, Kind::Layer] { + let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); + } + } + + /// Wait until queued writes are on disk. + pub fn flush(&self) { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + #[cfg(target_arch = "wasm32")] + { + q.jobs.clear(); + } + while !q.jobs.is_empty() || q.busy { + q = match cv.wait(q) { + Ok(q) => q, + Err(_) => return, + }; + } + } + + fn read(&self, kind: Kind, key: u128) -> Option> { + if !self.contains(kind, key) { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + } + let path = self.path(kind, key); + let data = std::fs::read(&path).ok(); + let ok = data.as_deref().and_then(|d| verify(d, kind)); + match ok { + Some(payload) => { + let payload = payload.to_vec(); + self.hits.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + if let Some(m) = g.map.get_mut(&(kind, key)) { + m.last_use = c; + } + } + // Persist the recency for the next session (best effort). + if let Ok(f) = std::fs::File::options().append(true).open(&path) { + let _ = f.set_modified(std::time::SystemTime::now()); + } + Some(payload) + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); + // Gone (another process evicted it) or damaged: forget it. + if data.is_some() { + let _ = std::fs::remove_file(&path); + } + self.forget(kind, key); + None + } + } + } + + fn forget(&self, kind: Kind, key: u128) { + if let Ok(mut g) = self.index.lock() + && let Some(m) = g.map.remove(&(kind, key)) + { + g.total -= m.bytes; + } + } + + fn enqueue(&self, kind: Kind, key: u128, payload: Vec) { + #[cfg(target_arch = "wasm32")] + { + self.write_now(kind, key, &payload); + } + #[cfg(not(target_arch = "wasm32"))] + { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + if q.jobs.len() >= QUEUE_LIMIT { + self.dropped.fetch_add(1, Ordering::Relaxed); + return; + } + q.jobs.push_back(Job::Write(kind, key, payload)); + cv.notify_all(); + } + } + + /// Write an entry now (atomic rename), then evict down to the limit. + fn write_now(&self, kind: Kind, key: u128, payload: &[u8]) { + if self.contains(kind, key) { + return; + } + let path = self.path(kind, key); + let Some(parent) = path.parent() else { return }; + if std::fs::create_dir_all(parent).is_err() { + return; + } + let mut data = Vec::with_capacity(payload.len() + 16); + data.extend_from_slice(MAGIC); + data.push(kind.tag()); + data.extend_from_slice(&[0, 0, 0]); + data.extend_from_slice(payload); + let sum = checksum(&data); + data.extend_from_slice(&sum.to_le_bytes()); + let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!("{key:032x}.tmp.{}.{n}", std::process::id())); + if std::fs::write(&tmp, &data).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + if std::fs::rename(&tmp, &path).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + self.writes.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + let bytes = data.len() as u64; + if let Some(old) = g.map.insert((kind, key), Meta { bytes, last_use: c }) { + g.total -= old.bytes; + } + g.total += bytes; + } + self.evict(); + } + + /// Remove least recently used entries until the cache fits its limit. + fn evict(&self) { + let max = self.max_bytes(); + let victims: Vec<(Kind, u128)> = { + let Ok(mut g) = self.index.lock() else { return }; + if g.total <= max { + return; + } + let mut by_age: Vec<(u64, (Kind, u128), u64)> = g.map.iter().map(|(k, m)| (m.last_use, *k, m.bytes)).collect(); + by_age.sort_unstable_by_key(|e| e.0); + let mut out = vec![]; + for (_, k, bytes) in by_age { + if g.total <= max { + break; + } + g.map.remove(&k); + g.total -= bytes; + out.push(k); + } + out + }; + for (kind, key) in victims { + let _ = std::fs::remove_file(self.path(kind, key)); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + } + + // ------------------------------------------------------------ typed entries + + pub fn get_frame(&self, key: u128) -> Option { + let p = self.read(Kind::Frame, key)?; + decode_frame(&p) + } + + /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). + pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 { + return; + } + self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)); + } + + pub fn get_layer(&self, key: u128) -> Option { + let p = self.read(Kind::Layer, key)?; + decode_layer(&p) + } + + /// Queue a layer buffer for writing. + pub fn put_layer(&self, key: u128, buf: &Buf) { + if self.contains(Kind::Layer, key) { + return; + } + self.enqueue(Kind::Layer, key, encode_layer(buf)); + } + + /// Write a layer buffer synchronously (tests). + pub fn put_layer_now(&self, key: u128, buf: &Buf) { + self.write_now(Kind::Layer, key, &encode_layer(buf)); + } + + /// Write a frame synchronously (tests). + pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { + let (m, cv) = &*q; + loop { + let job = { + let Ok(mut g) = m.lock() else { return }; + loop { + if let Some(j) = g.jobs.pop_front() { + g.busy = true; + break j; + } + if dc.strong_count() == 0 { + return; + } + g = match cv.wait_timeout(g, std::time::Duration::from_millis(500)) { + Ok((g, _)) => g, + Err(_) => return, + }; + } + }; + let Some(cache) = dc.upgrade() else { return }; + match job { + Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload), + } + drop(cache); + if let Ok(mut g) = m.lock() { + g.busy = false; + } + cv.notify_all(); + } +} + +/// Check magic, kind and checksum; returns the payload. +fn verify(d: &[u8], kind: Kind) -> Option<&[u8]> { + if d.len() < 16 || &d[..4] != MAGIC || d[4] != kind.tag() { + return None; + } + let (body, sum) = d.split_at(d.len() - 8); + let mut s = [0u8; 8]; + s.copy_from_slice(sum); + (checksum(body) == u64::from_le_bytes(s)).then_some(&body[8..]) +} + +fn encode_frame(w: u32, h: u32, rgba: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgba.len() / 2 + 16); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(rgba)); + out +} + +fn decode_frame(p: &[u8]) -> Option { + if p.len() < 8 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let rgba = lz4_flex::decompress_size_prepended(&p[8..]).ok()?; + (rgba.len() == w as usize * h as usize * 4).then_some(Frame8 { width: w, height: h, rgba }) +} + +fn encode_layer(b: &Buf) -> Vec { + let (w, h) = (b.img.width, b.img.height); + let n = b.img.data.len(); + // Byte planes: all first bytes of every float, then all second bytes… (compresses well). + let mut planes = vec![0u8; n * 16]; + for (i, px) in b.img.data.iter().enumerate() { + for c in 0..4 { + let bytes = px[c].to_le_bytes(); + for (k, byte) in bytes.iter().enumerate() { + planes[(k * 4 + c) * n + i] = *byte; + } + } + } + let mut out = Vec::with_capacity(planes.len() / 4 + 40); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&b.offset[0].to_le_bytes()); + out.extend_from_slice(&b.offset[1].to_le_bytes()); + out.extend_from_slice(&b.scale.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(&planes)); + out +} + +fn decode_layer(p: &[u8]) -> Option { + if p.len() < 32 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let f = |i: usize| -> Option { Some(f64::from_le_bytes(p[i..i + 8].try_into().ok()?)) }; + let offset = [f(8)?, f(16)?]; + let scale = f(24)?; + let planes = lz4_flex::decompress_size_prepended(&p[32..]).ok()?; + let n = w as usize * h as usize; + if planes.len() != n * 16 { + return None; + } + let mut img = Image::new(w, h); + for (i, px) in img.data.iter_mut().enumerate() { + for c in 0..4 { + let mut bytes = [0u8; 4]; + for (k, byte) in bytes.iter_mut().enumerate() { + *byte = planes[(k * 4 + c) * n + i]; + } + px[c] = f32::from_le_bytes(bytes); + } + } + Some(Buf { img, offset, scale }) +} + +// ---------------------------------------------------------------- content keys + +/// Bump when rendering changes in a way that makes old disk entries wrong. +const RENDER_VERSION: &str = "effectcraft-render-1"; + +fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { + h.write(format!("{v:?}").as_bytes()); + h.write(&[0xff]); +} + +/// A file's size and modification time (so edited footage gets new keys). +fn file_stamp(h: &mut Hash128, path: &str) { + if path.is_empty() { + return; + } + if let Ok(md) = std::fs::metadata(path) { + h.write_u64(md.len()); + if let Ok(t) = md.modified() + && let Ok(d) = t.duration_since(std::time::UNIX_EPOCH) + { + h.write_u64(d.as_nanos() as u64); + } + } +} + +fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) { + file_stamp(h, &f.path); + for p in &f.sequence { + file_stamp(h, p); + } +} + +/// Salt for layer cache keys on disk: the renderer version and every footage file's identity +/// (layer keys name footage items, not file contents). +pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + for (id, it) in &project.items { + if let effectcraft_project::ItemKind::Footage(f) = &it.kind { + h.write_u64(id.0); + hash_debug(&mut h, f); + footage_stamp(&mut h, f); + } + } + let v = h.finish(); + (v as u64) ^ ((v >> 64) as u64) +} + +/// Content key of a composition's pixels: project render settings, the comp and everything it +/// uses (nested comps, footage, solids, footage files' size and time). With expressions in play +/// (which can read any comp) the whole project counts. +pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { + use effectcraft_project::{ItemKind, LayerSource}; + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + hash_debug(&mut h, &project.settings); + let mut seen = std::collections::BTreeSet::new(); + let mut stack = vec![comp]; + let mut expressions = false; + while let Some(id) = stack.pop() { + if !seen.insert(id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { + stack.push(item); + } + if !expressions { + l.props.walk("", &mut |_, p| expressions |= p.expr.is_some()); + } + } + } + } + let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; + for id in ids { + let Some(it) = project.item(id) else { continue }; + h.write_u64(id.0); + h.write(it.name.as_bytes()); + hash_debug(&mut h, &it.kind); + if let ItemKind::Footage(f) = &it.kind { + footage_stamp(&mut h, f); + } + } + h.write_u64(comp.0); + h.finish() +} + +/// Disk key of one viewer frame. +pub fn frame_key(content: u128, frame: i64, scale_milli: u32, view: u64) -> u128 { + let mut h = Hash128::default(); + h.write(&content.to_le_bytes()); + h.write_u64(frame as u64); + h.write_u64(scale_milli as u64); + h.write_u64(view); + h.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmpdir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-disk-cache-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + fn buf(seed: u32, w: u32, h: u32) -> Buf { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + let v = ((i as u32).wrapping_mul(2654435761u32).wrapping_add(seed) % 1000) as f32 / 999.0; + *p = [v, v * 0.5, 1.0 - v, if i % 7 == 0 { 0.0 } else { 1.0 }]; + } + Buf { img, offset: [1.5, -2.25], scale: 0.5 } + } + + #[test] + fn hit_miss_and_exact_round_trip() { + let dir = tmpdir("hit"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert!(dc.get_layer(42).is_none()); + let b = buf(1, 33, 17); + dc.put_layer(42, &b); + dc.flush(); + let back = dc.get_layer(42).unwrap(); + assert_eq!(back.img, b.img); + assert_eq!((back.offset, back.scale), (b.offset, b.scale)); + let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); + dc.put_frame(7, 20, 10, &rgba); + dc.flush(); + assert_eq!(dc.get_frame(7).unwrap().rgba, rgba); + let st = dc.stats(); + assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2)); + // Wrong kind is a miss. + assert!(dc.get_frame(42).is_none()); + } + + #[test] + fn persists_across_restarts_and_clears() { + let dir = tmpdir("persist"); + { + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_layer_now(1, &buf(1, 8, 8)); + dc.put_frame_now(2, 2, 2, &[9; 16]); + } + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.stats().entries, 2); + assert!(dc.contains(Kind::Layer, 1) && dc.contains(Kind::Frame, 2)); + assert_eq!(dc.get_frame(2).unwrap().rgba, vec![9; 16]); + dc.clear(); + assert_eq!(dc.stats().entries, 0); + assert!(dc.get_frame(2).is_none()); + let again = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(again.stats().entries, 0); + } + + #[test] + fn lru_eviction_respects_the_limit() { + let dir = tmpdir("lru"); + let dc = DiskCache::open(&dir, u64::MAX).unwrap(); + for k in 0..6u128 { + dc.put_layer_now(k, &buf(k as u32, 64, 64)); + } + let per = dc.stats().bytes / 6; + // Touch 0 so it is the most recently used. + assert!(dc.get_layer(0).is_some()); + dc.set_max_bytes(per * 3 + per / 2); + let st = dc.stats(); + assert!(st.bytes <= per * 3 + per / 2, "{st:?}"); + assert_eq!(st.entries, 3); + assert!(dc.contains(Kind::Layer, 0), "recently used survives"); + assert!(!dc.contains(Kind::Layer, 1) && !dc.contains(Kind::Layer, 2) && !dc.contains(Kind::Layer, 3)); + assert_eq!(st.evictions, 3); + // Files are really gone. + let files = walk(&dir); + assert_eq!(files, 3); + // New writes keep evicting. + dc.put_layer_now(99, &buf(9, 64, 64)); + assert!(dc.stats().bytes <= per * 3 + per / 2); + } + + fn walk(d: &Path) -> usize { + let mut n = 0; + if let Ok(rd) = std::fs::read_dir(d) { + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + n += walk(&p); + } else if p.extension().is_some_and(|x| x == "ecc") { + n += 1; + } + } + } + n + } + + #[test] + fn damaged_files_are_misses() { + let dir = tmpdir("damaged"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_frame_now(5, 4, 4, &[1; 64]); + let p = dc.path(Kind::Frame, 5); + let mut d = std::fs::read(&p).unwrap(); + let n = d.len(); + d[n / 2] ^= 0xff; + std::fs::write(&p, d).unwrap(); + assert!(dc.get_frame(5).is_none()); + assert!(!dc.contains(Kind::Frame, 5)); + assert!(!p.exists()); + } + + #[test] + fn concurrent_readers_and_writers() { + let dir = tmpdir("concurrent"); + let a = DiskCache::open(&dir, 1 << 30).unwrap(); + // A second instance on the same folder (another process, in effect). + let b = DiskCache::open(&dir, 1 << 30).unwrap(); + std::thread::scope(|s| { + for t in 0..4u32 { + let (a, b) = (a.clone(), b.clone()); + s.spawn(move || { + for i in 0..20u128 { + let k = i % 5; + let w = if t % 2 == 0 { &a } else { &b }; + w.put_layer_now(k, &buf(k as u32, 16, 16)); + if let Some(x) = w.get_layer(k) { + assert_eq!(x.img, buf(k as u32, 16, 16).img); + } + } + }); + } + }); + b.rescan(); + for k in 0..5u128 { + assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 01f5f96af..ff71fb575 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio; pub mod cache; pub mod color; +pub mod disk_cache; use color::Region; pub mod eval; pub mod masks; @@ -56,6 +57,16 @@ pub trait FootageSource: Send + Sync { fn cache_budget(&self) -> Option { None } + /// Vector footage (SVG) rasterised at `scale` × its pixel size, for Continuously Rasterize. + /// `None` when the footage is not vector or cannot be read. + fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option> { + None + } +} + +/// Footage that can be rasterised at any scale (SVG). +pub fn is_vector_footage(f: &Footage) -> bool { + f.codec == "SVG" } /// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]). @@ -418,7 +429,12 @@ impl<'a> Renderer<'a> { /// stays sharp when scaled up). pub fn raster_scale(&self, ctx: &EvalCtx, layer: &Layer) -> f64 { let s = self.opts.scale; - if !layer.switches.collapse || !matches!(layer.source, LayerSource::Text | LayerSource::Shape) { + let vector = match layer.source { + LayerSource::Text | LayerSource::Shape => true, + LayerSource::Footage { item } => matches!(self.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if is_vector_footage(f)), + _ => false, + }; + if !layer.switches.collapse || !vector { return s; } let (l2c, _) = ctx.layer_to_comp(layer); @@ -723,6 +739,17 @@ impl<'a> Renderer<'a> { if !f.has_video { return None; } + // Continuously rasterised vector footage: drawn at the on-screen scale. + if layer.switches.collapse && is_vector_footage(f) { + let k = self.raster_scale(ctx, layer); + if let Some(img) = self.footage.vector_frame(*item, f, k) { + let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + return Some(buf); + } + } let lt = ctx.source_time(layer); // The proxy is decoded instead, at its own size, and fills the footage's frame. let pf = self.proxy_for(*item).unwrap_or(f); diff --git a/crates/script/Cargo.toml b/crates/script/Cargo.toml new file mode 100644 index 000000000..ef756107c --- /dev/null +++ b/crates/script/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "effectcraft-script" +description = "EffectCraft scripting: JavaScript (boa) with an After Effects-style object model (app, project, CompItem, Layer, Property…) over the engine's undoable commands" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-engine = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-keyframe = { workspace = true } +effectcraft-time = { workspace = true } +effectcraft-render = { workspace = true } +effectcraft-path = { workspace = true } +serde_json = { workspace = true } +boa_engine = { workspace = true } + +# On the web, boa's clock and Math.random need the browser (getrandom `wasm_js`, web-time). +[target.'cfg(target_arch = "wasm32")'.dependencies] +boa_engine = { workspace = true, features = ["js"] } + +[dev-dependencies] +effectcraft-expr = { workspace = true } diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs new file mode 100644 index 000000000..a1e9e5d44 --- /dev/null +++ b/crates/script/src/lib.rs @@ -0,0 +1,64 @@ +//! # effectcraft-script (L4) +//! +//! Scripting with an After Effects-style object model, on the boa JavaScript engine (the one the +//! expressions use). Scripts written for After Effects' documented scripting API mostly run +//! unchanged: +//! +//! ```text +//! app.beginUndoGroup("Build"); +//! var comp = app.project.items.addComp("Main", 1920, 1080, 1, 10, 30); +//! var solid = comp.layers.addSolid([1, 0, 0], "Red", 1920, 1080, 1); +//! solid.property("ADBE Transform Group").property("ADBE Position").setValueAtTime(0, [0, 540]); +//! solid.Effects.addProperty("ADBE Gaussian Blur 2").property("Blurriness").setValue(20); +//! app.endUndoGroup(); +//! ``` +//! +//! * **Object model** (`prelude.js`): `app` (project, open/newProject, undo groups, +//! `executeCommand`/`findMenuCommandId`, `scheduleTask`), `Project`, `ItemCollection`, +//! `CompItem`/`FootageItem`/`FolderItem`, `LayerCollection`, `AVLayer`/`TextLayer`/ +//! `ShapeLayer`/`CameraLayer`/`LightLayer`, `PropertyGroup`/`Property` (values, keyframes, +//! eases, expressions), `MaskPropertyGroup`, `TextDocument`, `Shape`, `KeyframeEase`, +//! `MarkerValue`, `RenderQueue`/`RenderQueueItem`/`OutputModule`, `ImportOptions`, `File`/ +//! `Folder`, `$`, `alert`/`writeLn`, and the enums (`BlendingMode`, `KeyframeInterpolationType`, +//! `TrackMatteType`, `LightType`, `ParagraphJustification`, `PropertyValueType`…). +//! * **Edits are engine commands**: every mutating call runs a command through +//! [`Session::execute`](effectcraft_engine::Session::execute), so it is undoable, journaled +//! and identical to the UI's action; `app.beginUndoGroup`/`endUndoGroup` fold everything in +//! between into one undo step. +//! * **Match names** ([`matchnames`]): After Effects' documented match names for our properties +//! and effects (`ADBE Transform Group`, `ADBE Position`, `ADBE Gaussian Blur 2`…). +//! * **Security**: scripts can import/open/save projects and render, but `File` reads are limited +//! to the project's folder and writes (and the network) are off unless Preferences ▸ Scripting +//! & Expressions ▸ Allow Scripts to Write Files and Access Network is on. +//! +//! Entry points: [`install`] sets [`Session::script`](effectcraft_engine::Session::script), which +//! the `script.run` command, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`), the Script Console +//! panel, `effectcraft-cli script` and the MCP `run_script` tool use. + +#![recursion_limit = "256"] + +pub mod matchnames; +mod model; +mod runtime; + +use effectcraft_engine::{ScriptRequest, Session}; + +pub use runtime::{Outcome, ScriptError, run}; + +/// The [`effectcraft_engine::ScriptRunner`] this crate provides. +pub fn runner(s: &mut Session, req: &ScriptRequest) -> serde_json::Value { + run(s, req).to_json() +} + +/// Enable scripting on a session. +pub fn install(s: &mut Session) { + s.script = Some(runner); +} + +/// Run `code` as a one-off script named `name`. +pub fn run_code(s: &mut Session, code: &str, name: &str) -> Outcome { + run(s, &ScriptRequest { code, name, console: false }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/script/src/matchnames.rs b/crates/script/src/matchnames.rs new file mode 100644 index 000000000..623027c13 --- /dev/null +++ b/crates/script/src/matchnames.rs @@ -0,0 +1,443 @@ +//! After Effects-compatible match names for our property tree and effects, so common scripts +//! (`layer.property("ADBE Transform Group").property("ADBE Position")`, +//! `layer.Effects.addProperty("ADBE Gaussian Blur 2")`) work unchanged. +//! +//! The names are the stable identifiers After Effects documents for scripting (Scripting Guide +//! ▸ match names). Properties without a documented equivalent keep our own match id +//! (`perChar3d`, `ec.blur.bilateral`), which `property()` and `addProperty()` accept too. + +use effectcraft_project::{LayerSource, Node}; + +/// Match name of a layer (its root property group). +pub fn layer(source: &LayerSource) -> &'static str { + match source { + LayerSource::Text => "ADBE Text Layer", + LayerSource::Shape => "ADBE Vector Layer", + LayerSource::Camera => "ADBE Camera Layer", + LayerSource::Light { .. } => "ADBE Light Layer", + _ => "ADBE AV Layer", + } +} + +/// Top-level groups of a layer. +const LAYER_GROUPS: &[(&str, &str)] = &[ + ("transform", "ADBE Transform Group"), + ("masks", "ADBE Mask Parade"), + ("effects", "ADBE Effect Parade"), + ("text", "ADBE Text Properties"), + ("contents", "ADBE Root Vectors Group"), + ("materialOptions", "ADBE Material Options Group"), + ("cameraOptions", "ADBE Camera Options Group"), + ("lightOptions", "ADBE Light Options Group"), + ("audio", "ADBE Audio Group"), + ("layerStyles", "ADBE Layer Styles"), + ("motionTrackers", "ADBE MTrackers"), + ("timeRemap", "ADBE Time Remapping"), +]; + +const TRANSFORM: &[(&str, &str)] = &[ + ("anchor", "ADBE Anchor Point"), + // Cameras and lights store the Point of Interest where AVLayers keep the Anchor Point. + ("poi", "ADBE Anchor Point"), + ("position", "ADBE Position"), + ("positionX", "ADBE Position_0"), + ("positionY", "ADBE Position_1"), + ("positionZ", "ADBE Position_2"), + ("scale", "ADBE Scale"), + ("orientation", "ADBE Orientation"), + ("rotationX", "ADBE Rotate X"), + ("rotationY", "ADBE Rotate Y"), + ("rotation", "ADBE Rotate Z"), + ("opacity", "ADBE Opacity"), +]; + +const MASK: &[(&str, &str)] = + &[("path", "ADBE Mask Shape"), ("feather", "ADBE Mask Feather"), ("opacity", "ADBE Mask Opacity"), ("expansion", "ADBE Mask Offset")]; + +const TEXT: &[(&str, &str)] = &[ + ("sourceText", "ADBE Text Document"), + ("pathOptions", "ADBE Text Path Options"), + ("moreOptions", "ADBE Text More Options"), + ("animators", "ADBE Text Animators"), +]; + +const TEXT_ANIMATOR: &[(&str, &str)] = &[("selectors", "ADBE Text Selectors"), ("properties", "ADBE Text Animator Properties")]; + +const CAMERA: &[(&str, &str)] = &[ + ("zoom", "ADBE Camera Zoom"), + ("dof", "ADBE Camera Depth of Field"), + ("focusDistance", "ADBE Camera Focus Distance"), + ("aperture", "ADBE Camera Aperture"), + ("blurLevel", "ADBE Camera Blur Level"), +]; + +const LIGHT: &[(&str, &str)] = &[ + ("intensity", "ADBE Light Intensity"), + ("color", "ADBE Light Color"), + ("coneAngle", "ADBE Light Cone Angle"), + ("coneFeather", "ADBE Light Cone Feather 2"), + ("falloff", "ADBE Light Falloff Type"), + ("radius", "ADBE Light Falloff Start"), + ("falloffDistance", "ADBE Light Falloff Distance"), + ("castsShadows", "ADBE Light Casts Shadows"), + ("shadowDarkness", "ADBE Light Shadow Darkness"), + ("shadowDiffusion", "ADBE Light Shadow Diffusion"), +]; + +const MATERIAL: &[(&str, &str)] = &[ + ("castsShadows", "ADBE Casts Shadows"), + ("lightTransmission", "ADBE Light Transmission"), + ("acceptsShadows", "ADBE Accepts Shadows"), + ("acceptsLights", "ADBE Accepts Lights"), + ("ambient", "ADBE Ambient Coefficient"), + ("diffuse", "ADBE Diffuse Coefficient"), + ("specularIntensity", "ADBE Specular Coefficient"), + ("specularShininess", "ADBE Shininess Coefficient"), + ("metal", "ADBE Metal Coefficient"), +]; + +/// Shape items (Contents ▸ Add): our match id → AE match name. +pub const SHAPE_ITEMS: &[(&str, &str)] = &[ + ("group", "ADBE Vector Group"), + ("rect", "ADBE Vector Shape - Rect"), + ("ellipse", "ADBE Vector Shape - Ellipse"), + ("star", "ADBE Vector Shape - Star"), + ("path", "ADBE Vector Shape - Group"), + ("fill", "ADBE Vector Graphic - Fill"), + ("stroke", "ADBE Vector Graphic - Stroke"), + ("gfill", "ADBE Vector Graphic - G-Fill"), + ("gstroke", "ADBE Vector Graphic - G-Stroke"), + ("merge", "ADBE Vector Filter - Merge"), + ("offset", "ADBE Vector Filter - Offset"), + ("pucker", "ADBE Vector Filter - PB"), + ("repeater", "ADBE Vector Filter - Repeater"), + ("round", "ADBE Vector Filter - RC"), + ("trim", "ADBE Vector Filter - Trim"), + ("twist", "ADBE Vector Filter - Twist"), + ("wiggle", "ADBE Vector Filter - Roughen"), + ("zigzag", "ADBE Vector Filter - Zigzag"), +]; + +/// Properties inside shape items: (item match id, property match id, AE match name). +const SHAPE_PROPS: &[(&str, &str, &str)] = &[ + ("group", "blend", "ADBE Vector Blend Mode"), + ("group", "contents", "ADBE Vectors Group"), + ("group", "transform", "ADBE Vector Transform Group"), + ("transform", "anchor", "ADBE Vector Anchor"), + ("transform", "position", "ADBE Vector Position"), + ("transform", "scale", "ADBE Vector Scale"), + ("transform", "skew", "ADBE Vector Skew"), + ("transform", "skewAxis", "ADBE Vector Skew Axis"), + ("transform", "rotation", "ADBE Vector Rotation"), + ("transform", "opacity", "ADBE Vector Group Opacity"), + ("rect", "direction", "ADBE Vector Shape Direction"), + ("rect", "size", "ADBE Vector Rect Size"), + ("rect", "position", "ADBE Vector Rect Position"), + ("rect", "roundness", "ADBE Vector Rect Roundness"), + ("ellipse", "direction", "ADBE Vector Shape Direction"), + ("ellipse", "size", "ADBE Vector Ellipse Size"), + ("ellipse", "position", "ADBE Vector Ellipse Position"), + ("star", "direction", "ADBE Vector Shape Direction"), + ("star", "type", "ADBE Vector Star Type"), + ("star", "points", "ADBE Vector Star Points"), + ("star", "position", "ADBE Vector Star Position"), + ("star", "rotation", "ADBE Vector Star Rotation"), + ("star", "innerRadius", "ADBE Vector Star Inner Radius"), + ("star", "outerRadius", "ADBE Vector Star Outer Radius"), + ("star", "innerRoundness", "ADBE Vector Star Inner Roundess"), + ("star", "outerRoundness", "ADBE Vector Star Outer Roundess"), + ("path", "direction", "ADBE Vector Shape Direction"), + ("path", "path", "ADBE Vector Shape"), + ("fill", "blend", "ADBE Vector Blend Mode"), + ("fill", "composite", "ADBE Vector Composite Order"), + ("fill", "rule", "ADBE Vector Fill Rule"), + ("fill", "color", "ADBE Vector Fill Color"), + ("fill", "opacity", "ADBE Vector Fill Opacity"), + ("stroke", "blend", "ADBE Vector Blend Mode"), + ("stroke", "composite", "ADBE Vector Composite Order"), + ("stroke", "color", "ADBE Vector Stroke Color"), + ("stroke", "opacity", "ADBE Vector Stroke Opacity"), + ("stroke", "width", "ADBE Vector Stroke Width"), + ("stroke", "cap", "ADBE Vector Stroke Line Cap"), + ("stroke", "join", "ADBE Vector Stroke Line Join"), + ("stroke", "miter", "ADBE Vector Stroke Miter Limit"), + ("stroke", "dashes", "ADBE Vector Stroke Dashes"), + ("trim", "start", "ADBE Vector Trim Start"), + ("trim", "end", "ADBE Vector Trim End"), + ("trim", "offset", "ADBE Vector Trim Offset"), + ("trim", "mode", "ADBE Vector Trim Type"), + ("repeater", "copies", "ADBE Vector Repeater Copies"), + ("repeater", "offset", "ADBE Vector Repeater Offset"), + ("repeater", "composite", "ADBE Vector Repeater Order"), + ("repeater", "transform", "ADBE Vector Repeater Transform"), + ("round", "radius", "ADBE Vector RoundCorner Radius"), + ("pucker", "amount", "ADBE Vector PuckerBloat Amount"), + ("twist", "angle", "ADBE Vector Twist Angle"), + ("twist", "center", "ADBE Vector Twist Center"), + ("merge", "mode", "ADBE Vector Merge Type"), +]; + +/// Effects: our effect id → AE match name (the documented ones; the rest keep our id). +pub const EFFECTS: &[(&str, &str)] = &[ + ("ec.blur.gaussian", "ADBE Gaussian Blur 2"), + ("ec.blur.fastbox", "ADBE Box Blur2"), + ("ec.blur.directional", "ADBE Motion Blur"), + ("ec.blur.radial", "ADBE Radial Blur"), + ("ec.blur.cameralens", "ADBE Camera Lens Blur"), + ("ec.blur.channel", "ADBE Channel Blur"), + ("ec.blur.compound", "ADBE Compound Blur"), + ("ec.blur.sharpen", "ADBE Sharpen"), + ("ec.blur.smart", "ADBE Smart Blur"), + ("ec.blur.unsharp", "ADBE Unsharp Mask2"), + ("ec.channel.invert", "ADBE Invert"), + ("ec.channel.setmatte", "ADBE Set Matte3"), + ("ec.channel.setchannels", "ADBE Set Channels"), + ("ec.channel.shiftchannels", "ADBE Shift Channels"), + ("ec.channel.minimax", "ADBE Minimax"), + ("ec.channel.blend", "ADBE Blend"), + ("ec.channel.arithmetic", "ADBE Arithmetic"), + ("ec.channel.calculations", "ADBE Calculations"), + ("ec.channel.solidcomposite", "ADBE Solid Composite"), + ("ec.color.brightnesscontrast", "ADBE Brightness & Contrast 2"), + ("ec.color.curves", "ADBE CurvesCustom"), + ("ec.color.levels", "ADBE Easy Levels2"), + ("ec.color.levelsic", "ADBE Pro Levels2"), + ("ec.color.huesaturation", "ADBE HUE SATURATION"), + ("ec.color.exposure", "ADBE Exposure2"), + ("ec.color.tint", "ADBE Tint"), + ("ec.color.tritone", "ADBE Tritone"), + ("ec.color.blackwhite", "ADBE Black&White"), + ("ec.color.channelmixer", "ADBE CHANNEL MIXER"), + ("ec.color.colorbalance", "ADBE Color Balance 2"), + ("ec.color.colorbalancehls", "ADBE Color Balance (HLS)"), + ("ec.color.changecolor", "ADBE Change Color"), + ("ec.color.changetocolor", "ADBE Change To Color"), + ("ec.color.colorama", "APC Colorama"), + ("ec.color.leavecolor", "ADBE Leave Color"), + ("ec.color.photofilter", "ADBE Photo Filter"), + ("ec.color.vibrance", "ADBE Vibrance"), + ("ec.color.lumetri", "ADBE Lumetri"), + ("ec.color.shadowhighlight", "ADBE Shadow/Highlight"), + ("ec.color.selectivecolor", "ADBE Selective Color"), + ("ec.color.gammapedestalgain", "ADBE Gamma/Pedestal/Gain"), + ("ec.color.equalize", "ADBE Equalize"), + ("ec.color.autocontrast", "ADBE AutoContrast"), + ("ec.color.autolevels", "ADBE AutoLevels"), + ("ec.color.autocolor", "ADBE AutoColor"), + ("ec.control.slider", "ADBE Slider Control"), + ("ec.control.angle", "ADBE Angle Control"), + ("ec.control.checkbox", "ADBE Checkbox Control"), + ("ec.control.color", "ADBE Color Control"), + ("ec.control.point", "ADBE Point Control"), + ("ec.control.point3d", "ADBE Point3D Control"), + ("ec.control.layer", "ADBE Layer Control"), + ("ec.control.dropdown", "ADBE Dropdown Control"), + ("ec.distort.transform", "ADBE Geometry2"), + ("ec.distort.cornerpin", "ADBE Corner Pin"), + ("ec.distort.bulge", "ADBE Bulge"), + ("ec.distort.mirror", "ADBE Mirror"), + ("ec.distort.offset", "ADBE Offset"), + ("ec.distort.ripple", "ADBE Ripple"), + ("ec.distort.spherize", "ADBE Spherize"), + ("ec.distort.twirl", "ADBE Twirl"), + ("ec.distort.wavewarp", "ADBE Wave Warp"), + ("ec.distort.turbulentdisplace", "ADBE Turbulent Displace"), + ("ec.distort.displacementmap", "ADBE Displacement Map"), + ("ec.distort.meshwarp", "ADBE MESH WARP"), + ("ec.distort.polar", "ADBE Polar Coordinates"), + ("ec.distort.opticscompensation", "ADBE Optics Compensation"), + ("ec.distort.magnify", "ADBE Magnify"), + ("ec.distort.warpstabilizer", "ADBE SubspaceStabilizer"), + ("ec.distort.puppet", "ADBE FreePin3"), + ("ec.generate.fill", "ADBE Fill"), + ("ec.generate.gradientramp", "ADBE Ramp"), + ("ec.generate.fourcolor", "ADBE 4ColorGradient"), + ("ec.generate.stroke", "ADBE Stroke"), + ("ec.generate.grid", "ADBE Grid"), + ("ec.generate.checkerboard", "ADBE Checkerboard"), + ("ec.generate.circle", "ADBE Circle"), + ("ec.generate.ellipse", "ADBE Ellipse"), + ("ec.generate.cellpattern", "ADBE Cell Pattern"), + ("ec.generate.lensflare", "ADBE Lens Flare"), + ("ec.generate.vegas", "ADBE Vegas"), + ("ec.generate.writeon", "ADBE Write On"), + ("ec.generate.audiospectrum", "ADBE AudSpect"), + ("ec.generate.audiowaveform", "ADBE AudWave"), + ("ec.generate.beam", "ADBE Laser"), + ("ec.generate.radiowaves", "ADBE Radio Waves"), + ("ec.generate.scribble", "ADBE Scribble Fill"), + ("ec.generate.paintbucket", "ADBE Paint Bucket"), + ("ec.key.colorkey", "ADBE Color Key"), + ("ec.key.luma", "ADBE Luma Key"), + ("ec.key.linearcolor", "ADBE Linear Color Key2"), + ("ec.key.colorrange", "ADBE Color Range"), + ("ec.key.colordifference", "ADBE Color Difference Key"), + ("ec.key.extract", "ADBE Extract"), + ("ec.key.spill", "ADBE Spill2"), + ("ec.matte.simplechoker", "ADBE Simple Choker"), + ("ec.matte.mattechoker", "ADBE Matte Choker"), + ("ec.noise.noise", "ADBE Noise"), + ("ec.noise.fractal", "ADBE Fractal Noise"), + ("ec.noise.median", "ADBE Median"), + ("ec.noise.addgrain", "VISINF Grain Implant"), + ("ec.noise.dustscratches", "ADBE Dust & Scratches"), + ("ec.noise.turbulent", "ADBE Turbulent Noise"), + ("ec.perspective.dropshadow", "ADBE Drop Shadow"), + ("ec.perspective.radialshadow", "ADBE Radial Shadow"), + ("ec.perspective.bevelalpha", "ADBE Bevel Alpha"), + ("ec.perspective.beveledges", "ADBE Bevel Edges"), + ("ec.perspective.3dglasses", "ADBE 3D Glasses2"), + ("ec.stylize.glow", "ADBE Glo2"), + ("ec.stylize.mosaic", "ADBE Mosaic"), + ("ec.stylize.posterize", "ADBE Posterize"), + ("ec.stylize.threshold", "ADBE Threshold2"), + ("ec.stylize.findedges", "ADBE Find Edges"), + ("ec.stylize.emboss", "ADBE Emboss"), + ("ec.stylize.coloremboss", "ADBE Color Emboss"), + ("ec.stylize.roughenedges", "ADBE Roughen Edges"), + ("ec.stylize.scatter", "ADBE Scatter"), + ("ec.stylize.strobe", "ADBE Strobe"), + ("ec.stylize.motiontile", "ADBE Tile"), + ("ec.stylize.texturize", "ADBE Texturize"), + ("ec.stylize.cartoon", "ADBE Cartoonify"), + ("ec.stylize.brushstrokes", "ADBE Brush Strokes"), + ("ec.text.numbers", "ADBE Numbers2"), + ("ec.text.timecode", "ADBE Timecode"), + ("ec.time.echo", "ADBE Echo"), + ("ec.time.posterizetime", "ADBE Posterize Time"), + ("ec.time.timedisplacement", "ADBE Time Displacement"), + ("ec.time.timedifference", "ADBE Time Difference"), + ("ec.time.pixelmotionblur", "ADBE Pixel Motion Blur"), + ("ec.time.timewarp", "ADBE Timewarp"), + ("ec.transition.linearwipe", "ADBE Linear Wipe"), + ("ec.transition.radialwipe", "ADBE Radial Wipe"), + ("ec.transition.venetian", "ADBE Venetian Blinds"), + ("ec.transition.blockdissolve", "ADBE Block Dissolve"), + ("ec.transition.gradientwipe", "ADBE Gradient Wipe"), + ("ec.transition.iriswipe", "ADBE Iris Wipe"), + ("ec.utility.applylut", "ADBE Apply Color LUT2"), + ("ec.utility.cineon", "ADBE Cineon Converter2"), + ("ec.utility.growbounds", "ADBE Grow Bounds"), +]; + +fn find<'a>(t: &'a [(&str, &'a str)], k: &str) -> Option<&'a str> { + t.iter().find(|(a, _)| *a == k).map(|(_, b)| *b) +} + +/// AE match name of an effect id (falls back to the id). +pub fn effect(id: &str) -> String { + find(EFFECTS, id).map(str::to_string).unwrap_or_else(|| id.to_string()) +} + +/// Our effect id for an AE effect match name. +pub fn effect_from_match(m: &str) -> Option<&'static str> { + EFFECTS.iter().find(|(_, b)| b.eq_ignore_ascii_case(m)).map(|(a, _)| *a) +} + +/// Our shape item kind (`layer.addShapeItem`) for an AE match name, our match id or a display +/// name (`Rectangle`, `Fill`, `Trim Paths`). +pub fn shape_kind(name: &str) -> Option<&'static str> { + if let Some((k, _)) = SHAPE_ITEMS.iter().find(|(k, m)| m.eq_ignore_ascii_case(name) || k.eq_ignore_ascii_case(name)) { + return Some(k); + } + let n: String = name.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase(); + Some(match n.as_str() { + "rectangle" | "rectanglepath" => "rect", + "ellipsepath" => "ellipse", + "polystar" | "polystarpath" => "star", + "polygon" => "polygon", + "group" | "vectorgroup" => "group", + "fill" => "fill", + "stroke" => "stroke", + "gradientfill" => "gfill", + "gradientstroke" => "gstroke", + "trimpaths" | "trim" => "trim", + "repeater" => "repeater", + "roundcorners" => "round", + "offsetpaths" => "offset", + "puckerbloat" | "puckerandbloat" => "pucker", + "twist" => "twist", + "zigzag" => "zigzag", + "wigglepaths" => "wiggle", + "mergepaths" => "merge", + _ => return None, + }) +} + +/// Match name of the node at the end of `chain` (layer root's children → … → node). `effect` +/// params are numbered `-0001…` in parameter order, like After Effects. +pub fn node(chain: &[&Node]) -> String { + let Some(last) = chain.last() else { return String::new() }; + let id = last.match_id(); + let parent = chain.len().checked_sub(2).map(|i| chain[i]); + let grand = chain.len().checked_sub(3).map(|i| chain[i]); + let top = chain.first().map(|n| n.match_id()).unwrap_or(""); + let own = || id.to_string(); + if chain.len() == 1 { + return find(LAYER_GROUPS, id).map(str::to_string).unwrap_or_else(own); + } + let pid = parent.map(|p| p.match_id()).unwrap_or(""); + match top { + "transform" if chain.len() == 2 => find(TRANSFORM, id).map(str::to_string).unwrap_or_else(own), + "masks" if chain.len() == 2 => "ADBE Mask Atom".into(), + "masks" if chain.len() == 3 => find(MASK, id).map(str::to_string).unwrap_or_else(own), + "effects" if chain.len() == 2 => effect(id), + "effects" if chain.len() == 3 => { + let fx = parent.and_then(Node::as_group); + match fx { + Some(g) => { + let i = g.children.iter().position(|c| c.uid() == last.uid()).unwrap_or(0) + 1; + format!("{}-{i:04}", effect(&g.match_id)) + } + None => own(), + } + } + "text" if chain.len() == 2 => find(TEXT, id).map(str::to_string).unwrap_or_else(own), + "text" if pid == "animators" => "ADBE Text Animator".into(), + "text" if grand.map(|g| g.match_id()) == Some("animators") => find(TEXT_ANIMATOR, id).map(str::to_string).unwrap_or_else(own), + "cameraOptions" => find(CAMERA, id).map(str::to_string).unwrap_or_else(own), + "lightOptions" => find(LIGHT, id).map(str::to_string).unwrap_or_else(own), + "materialOptions" => find(MATERIAL, id).map(str::to_string).unwrap_or_else(own), + "audio" if id == "levels" => "ADBE Audio Levels".into(), + "contents" => { + // Items sit in a `contents` list; their properties under the item. + if pid == "contents" { + if id == "star" || id == "polygon" { + return "ADBE Vector Shape - Star".into(); + } + return find(SHAPE_ITEMS, id).map(str::to_string).unwrap_or_else(own); + } + if let Some((_, _, m)) = SHAPE_PROPS.iter().find(|(item, prop, _)| *item == pid && *prop == id) { + return m.to_string(); + } + own() + } + _ => own(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effect_names_round_trip() { + for (id, m) in EFFECTS { + assert_eq!(effect(id), *m); + assert_eq!(effect_from_match(m), Some(*id)); + } + assert_eq!(effect("ec.blur.ccvector"), "ec.blur.ccvector"); + let mut ids: Vec<&str> = EFFECTS.iter().map(|(a, _)| *a).collect(); + ids.sort(); + ids.dedup(); + assert_eq!(ids.len(), EFFECTS.len(), "duplicate effect ids"); + } + + #[test] + fn shape_kinds() { + assert_eq!(shape_kind("ADBE Vector Shape - Rect"), Some("rect")); + assert_eq!(shape_kind("Trim Paths"), Some("trim")); + assert_eq!(shape_kind("ADBE Vector Graphic - Fill"), Some("fill")); + assert_eq!(shape_kind("nope"), None); + } +} diff --git a/crates/script/src/model.rs b/crates/script/src/model.rs new file mode 100644 index 000000000..d1dd47683 --- /dev/null +++ b/crates/script/src/model.rs @@ -0,0 +1,679 @@ +//! Read-only queries behind the object model (`__query(kind, json)`): project, items, layers, +//! property nodes, values and keyframes as JSON shaped for the prelude. Every edit goes through +//! engine commands instead (`__exec`), so it is undoable and journaled. + +use effectcraft_engine::Session; +use effectcraft_engine::color::Label; +use effectcraft_keyframe::{Interp, TextDoc, Value as KV}; +use effectcraft_project::{Comp, GroupKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Node, ParamUi, PropGroup}; +use effectcraft_render::EvalCtx; +use effectcraft_time::Tick; +use serde_json::{Value as J, json}; + +use crate::matchnames; + +type R = Result; + +fn u(a: &J, k: &str) -> Option { + a.get(k).and_then(J::as_u64) +} +fn f(a: &J, k: &str) -> Option { + a.get(k).and_then(J::as_f64) +} + +fn comp_of<'a>(s: &'a Session, a: &J) -> Result<(ItemId, &'a Comp), String> { + let id = ItemId(u(a, "comp").ok_or("missing comp")?); + s.project.comp(id).map(|c| (id, c)).ok_or_else(|| "the composition no longer exists".to_string()) +} + +fn layer_of<'a>(s: &'a Session, a: &J) -> Result<(ItemId, &'a Comp, &'a Layer), String> { + let (cid, c) = comp_of(s, a)?; + let l = c.layer(LayerId(u(a, "layer").ok_or("missing layer")?)).ok_or("the layer no longer exists (it was deleted)")?; + Ok((cid, c, l)) +} + +pub fn query(s: &mut Session, kind: &str, a: &J) -> R { + match kind { + "project" => Ok(project(s)), + "item" => item(s, a), + "layer" => layer(s, a), + "node" => node(s, a), + "child" => child(s, a), + "value" => value(s, a), + "keys" => keys(s, a), + "textdoc" => textdoc(s, a), + "markers" => markers(s, a), + "sourceRect" => source_rect(s, a), + "resolveAdd" => resolve_add(s, a), + "menuCommandId" => Ok(menu_command_id(a.get("name").and_then(J::as_str).unwrap_or(""))), + "commandAt" => { + let i = u(a, "index").unwrap_or(0) as usize; + Ok(effectcraft_engine::command_specs().get(i.wrapping_sub(1)).map(|c| json!(c.id)).unwrap_or(J::Null)) + } + "exprError" => { + let code = a.get("code").and_then(J::as_str).unwrap_or(""); + Ok(match s.expr_check { + Some(check) => json!(check(code).err().unwrap_or_default()), + None => json!(""), + }) + } + "labels" => Ok(json!(Label::ALL.iter().map(|l| l.name()).collect::>())), + "effects" => Ok(json!( + effectcraft_engine::effects::registry() + .iter() + .map(|e| json!({"id": e.id, "name": e.name, "matchName": matchnames::effect(e.id), "category": e.category})) + .collect::>() + )), + _ => Err(format!("unknown query `{kind}`")), + } +} + +fn project(s: &Session) -> J { + let p = &s.project; + let items: Vec = p.items.keys().map(|k| k.0).collect(); + let active = s.active_comp_id().map(|c| c.0).or_else(|| (s.state.project_selection.len() == 1).then(|| s.state.project_selection[0].0)); + json!({ + "items": items, + "active": active, + "selection": s.state.project_selection.iter().map(|i| i.0).collect::>(), + "path": s.path, + "dirty": s.is_dirty(), + "bitsPerChannel": p.settings.bit_depth.bits(), + "linearBlending": p.settings.blend_linear, + "linearizeWorkingSpace": p.settings.linearize, + "workingSpace": p.settings.working_space.map(|w| format!("{w:?}")).unwrap_or_default(), + "timeDisplayFrames": p.settings.time_display == effectcraft_project::TimeDisplayStyle::Frames, + "renderQueue": p.render_queue.len(), + "revision": s.revision, + }) +} + +fn label_index(l: Label) -> usize { + Label::ALL.iter().position(|x| *x == l).unwrap_or(0) +} + +fn item(s: &Session, a: &J) -> R { + let id = ItemId(u(a, "id").ok_or("missing id")?); + let it = s.project.item(id).ok_or("the item no longer exists (it was deleted)")?; + let mut o = json!({ + "id": id.0, + "name": it.name, + "typeName": it.type_name(), + "parent": it.parent.map(|p| p.0).unwrap_or(0), + "comment": it.comment, + "label": label_index(it.label), + "selected": s.state.project_selection.contains(&id), + }); + let used_in: Vec = s.project.comps().filter(|(_, c)| c.layers.iter().any(|l| l.source.item() == Some(id))).map(|(cid, _)| cid.0).collect(); + o["usedIn"] = json!(used_in); + match &it.kind { + ItemKind::Folder => { + o["kind"] = json!("folder"); + o["children"] = json!(s.project.items.values().filter(|c| c.parent == Some(id)).map(|c| c.id.0).collect::>()); + } + ItemKind::Comp(c) => { + let cid = id; + o["kind"] = json!("comp"); + let sel: Vec = s.state.selected_layers.iter().filter(|l| c.layer(**l).is_some()).map(|l| l.0).collect(); + let fps = c.frame_rate.as_f64(); + if let Some(m) = o.as_object_mut() { + m.extend( + json!({ + "width": c.width, "height": c.height, "pixelAspect": c.pixel_aspect, + "duration": c.duration.seconds(), "frameRate": fps, "frameDuration": c.frame_duration().seconds(), + "layers": c.layers.iter().map(|l| l.id.0).collect::>(), + "selectedLayers": sel, + "selectedProperties": s.state.selected_props.iter().filter(|(l, _)| c.layer(*l).is_some()).map(|(l, p)| json!([l.0, p])).collect::>(), + "time": s.time_of(cid).seconds(), + "workAreaStart": c.work_area.0.seconds(), "workAreaDuration": (c.work_area.1 - c.work_area.0).seconds(), + "bgColor": c.background, "displayStartTime": c.display_start.seconds(), + "hideShyLayers": c.hide_shy, "motionBlur": c.enable_motion_blur, "frameBlending": c.enable_frame_blending, + "draft3d": c.draft_3d, "shutterAngle": c.shutter_angle, "shutterPhase": c.shutter_phase, + "motionBlurSamplesPerFrame": c.motion_blur_samples, + "renderer": match c.renderer { effectcraft_project::Renderer::Classic3D => "ADBE Classic 3D", effectcraft_project::Renderer::Advanced3D => "ADBE Advanced 3d" }, + "open": s.state.open_comps.contains(&cid), "active": s.active_comp_id() == Some(cid), + "hasVideo": true, "hasAudio": c.layers.iter().any(|l| l.switches.audio && matches!(l.source, LayerSource::Footage { .. } | LayerSource::Comp { .. })), + "posterTime": c.poster_time.seconds(), + }) + .as_object() + .cloned() + .unwrap_or_default(), + ) + } + } + ItemKind::Footage(fo) => { + o["kind"] = json!("footage"); + o["file"] = json!(fo.path); + o["width"] = json!(fo.width); + o["height"] = json!(fo.height); + o["pixelAspect"] = json!(fo.pixel_aspect); + o["duration"] = json!(if fo.kind == effectcraft_project::FootageKind::Still { 0.0 } else { fo.duration.seconds() }); + o["frameRate"] = json!(if fo.kind == effectcraft_project::FootageKind::Still { 0.0 } else { fo.frame_rate.as_f64() }); + o["hasVideo"] = json!(fo.has_video); + o["hasAudio"] = json!(fo.has_audio); + o["footageMissing"] = json!(fo.missing); + o["isStill"] = json!(fo.kind == effectcraft_project::FootageKind::Still); + o["loop"] = json!(fo.loop_count); + } + ItemKind::Solid(so) => { + o["kind"] = json!("solid"); + o["width"] = json!(so.width); + o["height"] = json!(so.height); + o["pixelAspect"] = json!(so.pixel_aspect); + o["color"] = json!(so.color); + o["duration"] = json!(0.0); + o["frameRate"] = json!(0.0); + o["hasVideo"] = json!(true); + o["hasAudio"] = json!(false); + o["isStill"] = json!(true); + } + } + Ok(o) +} + +fn layer(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let kind = match &l.source { + LayerSource::Text => "text", + LayerSource::Shape => "shape", + LayerSource::Camera => "camera", + LayerSource::Light { .. } => "light", + _ => "av", + }; + let (w, h) = effectcraft_render::source_size(&s.project, l); + let t = s.time_of(cid); + Ok(json!({ + "id": l.id.0, + "comp": cid.0, + "kind": kind, + "name": l.name, + "index": c.index_of(l.id).unwrap_or(0), + "matchName": matchnames::layer(&l.source), + "root": l.props.uid, + "source": l.source.item().map(|i| i.0), + "nullLayer": matches!(l.source, LayerSource::Null), + "lightType": match &l.source { LayerSource::Light { kind } => kind.label(), _ => "" }, + "enabled": l.switches.video, + "audioEnabled": l.switches.audio, + "solo": l.switches.solo, + "locked": l.switches.locked, + "shy": l.switches.shy, + "collapse": l.switches.collapse, + "quality": format!("{:?}", l.switches.quality), + "samplingQuality": format!("{:?}", l.switches.sampling), + "effectsActive": l.switches.effects, + "frameBlending": format!("{:?}", l.switches.frame_blend), + "motionBlur": l.switches.motion_blur, + "adjustmentLayer": l.switches.adjustment, + "threeDLayer": l.switches.three_d, + "guideLayer": l.switches.guide, + "preserveTransparency": l.preserve_transparency, + "inPoint": l.in_point.seconds(), + "outPoint": l.out_point.seconds(), + "startTime": l.start_time.seconds(), + "stretch": l.stretch, + "time": l.layer_time(t).seconds(), + "active": l.switches.video && l.is_active_at(t), + "parent": l.parent.map(|p| p.0), + "blendingMode": l.blend_mode.label(), + "trackMatte": l.track_matte.map(|m| json!({"layer": m.layer.0, "kind": format!("{:?}", m.kind)})), + "isTrackMatte": c.layers.iter().any(|x| x.track_matte.is_some_and(|m| m.layer == l.id)), + "label": label_index(l.label), + "comment": l.comment, + "selected": s.state.selected_layers.contains(&l.id), + "autoOrient": format!("{:?}", l.auto_orient), + "width": w, "height": h, + "hasVideo": l.source.is_av(), + "hasAudio": matches!(l.source, LayerSource::Footage { .. } | LayerSource::Comp { .. }) && l.props.sub("audio").is_some(), + "timeRemapEnabled": l.props.prop("timeRemap").is_some(), + })) +} + +/// Node at the end of a uid chain below the layer root, and the chain itself. +fn chain(l: &Layer, uid: u64) -> Option> { + l.props.node_chain(uid) +} + +/// PropertyValueType of a property. +fn value_type(p: &effectcraft_project::Property) -> &'static str { + if matches!(p.ui, ParamUi::Mask) { + return "MASK_INDEX"; + } + match &p.value { + KV::Scalar(_) | KV::Bool(_) | KV::Enum(_) => "OneD", + KV::Vec2(_) => { + if p.spatial { + "TwoD_SPATIAL" + } else { + "TwoD" + } + } + KV::Vec3(_) => { + if p.spatial { + "ThreeD_SPATIAL" + } else { + "ThreeD" + } + } + KV::Color(_) => "COLOR", + KV::Path(_) => "SHAPE", + KV::Text(_) => "TEXT_DOCUMENT", + KV::Layer(_) => "LAYER_INDEX", + KV::Gradient(_) | KV::Str(_) => "CUSTOM_VALUE", + } +} + +/// Groups whose children are instances (effects, masks, shape items, animators). +fn indexed_parade(g: &PropGroup) -> bool { + matches!(g.match_id.as_str(), "effects" | "masks" | "contents" | "animators" | "selectors" | "layerStyles" | "motionTrackers") +} + +fn node_json(c: &Comp, l: &Layer, uid: u64) -> R { + if uid == 0 || uid == l.props.uid { + let n = l.props.children.len(); + return Ok(json!({ + "uid": l.props.uid, "kind": "group", "name": l.name, "matchName": matchnames::layer(&l.source), "matchId": "layer", + "index": c.index_of(l.id).unwrap_or(0), "depth": 0, "parent": null, "isLayer": true, "numProperties": n, + "children": l.props.children.iter().map(Node::uid).collect::>(), "propertyType": "NAMED_GROUP", + "enabled": l.switches.video, "canSetEnabled": true, + })); + } + let ch = chain(l, uid).ok_or("the property no longer exists (it was deleted)")?; + let node = *ch.last().ok_or("bad property")?; + let parent_group: &PropGroup = if ch.len() >= 2 { ch[ch.len() - 2].as_group().ok_or("bad tree")? } else { &l.props }; + let index = parent_group.children.iter().position(|x| x.uid() == uid).map(|i| i + 1).unwrap_or(0); + let parent_uid = if ch.len() >= 2 { ch[ch.len() - 2].uid() } else { l.props.uid }; + let match_name = matchnames::node(&ch); + let instance_parent = indexed_parade(parent_group) && ch.len() >= 2 || (ch.len() == 2 && ch[0].match_id() == "contents"); + let mut o = json!({ + "uid": uid, "name": node.name(), "matchName": match_name, "matchId": node.match_id(), "index": index, + "depth": ch.len(), "parent": parent_uid, "isLayer": false, + "isEffect": matches!(node.as_group().map(|g| &g.kind), Some(GroupKind::Effect { .. })), + "isMask": matches!(node.as_group().map(|g| &g.kind), Some(GroupKind::Mask { .. })), + }); + match node { + Node::Group(g) => { + let instance = matches!(g.kind, GroupKind::Indexed | GroupKind::Effect { .. } | GroupKind::Mask { .. } | GroupKind::Tracker { .. }); + o["kind"] = json!("group"); + o["numProperties"] = json!(g.children.len()); + o["children"] = json!(g.children.iter().map(Node::uid).collect::>()); + o["propertyType"] = json!(if indexed_parade(g) { "INDEXED_GROUP" } else { "NAMED_GROUP" }); + o["enabled"] = json!(g.enabled); + o["canSetEnabled"] = json!(instance); + o["instance"] = json!(instance || instance_parent); + o["effectId"] = json!(match &g.kind { + GroupKind::Effect { effect } => Some(effect.clone()), + _ => None, + }); + if let GroupKind::Mask { mode, inverted, color, locked, .. } = &g.kind { + o["maskMode"] = json!(mode.label()); + o["inverted"] = json!(inverted); + o["color"] = json!([color[0] as f64 / 255.0, color[1] as f64 / 255.0, color[2] as f64 / 255.0]); + o["locked"] = json!(locked); + } + } + Node::Prop(p) => { + o["kind"] = json!("prop"); + o["propertyType"] = json!("PROPERTY"); + o["valueType"] = json!(value_type(p)); + o["type"] = json!(p.value.kind_name()); + o["dims"] = json!(p.value.dims()); + o["spatial"] = json!(p.spatial); + o["numKeys"] = json!(p.keys.len()); + o["expression"] = json!(p.expr.as_ref().map(|e| e.text.clone()).unwrap_or_default()); + o["expressionEnabled"] = json!(p.has_expression()); + o["canVaryOverTime"] = json!(!p.static_only); + o["canSetExpression"] = json!(!p.static_only && !matches!(p.ui, ParamUi::Hidden)); + o["hidden"] = json!(matches!(p.ui, ParamUi::Hidden)); + o["enabled"] = json!(true); + o["canSetEnabled"] = json!(false); + o["isModified"] = json!(p.is_animated() || p.has_expression()); + if let ParamUi::Slider { min, max, .. } = p.ui { + o["min"] = json!(min); + o["max"] = json!(max); + } + if let ParamUi::Popup { options } = &p.ui { + o["options"] = json!(options); + } + o["ui"] = json!(match &p.ui { + ParamUi::Percent => "%", + ParamUi::Angle => "degrees", + ParamUi::Pixels | ParamUi::Point | ParamUi::Point3 => "pixels", + _ => "", + }); + // Position with Separate Dimensions. + o["separated"] = json!(node.match_id() == "position" && parent_group.match_id == "transform" && parent_group.get("positionX").is_some()); + o["separationFollower"] = json!(matches!(node.match_id(), "positionX" | "positionY" | "positionZ")); + } + } + Ok(o) +} + +fn node(s: &Session, a: &J) -> R { + let (_, c, l) = layer_of(s, a)?; + node_json(c, l, u(a, "uid").unwrap_or(0)) +} + +/// Normalized name for loose lookups: `Anchor Point`, `anchorPoint`, `anchor_point` → `anchorpoint`. +fn loose(s: &str) -> String { + s.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase() +} + +/// AE-style aliases for our match ids (attribute names scripts use). +fn alias(k: &str) -> Option<&'static str> { + Some(match k { + "anchorpoint" => "anchor", + "pointofinterest" => "poi", + "xrotation" => "rotationX", + "yrotation" => "rotationY", + "zrotation" => "rotation", + "xposition" => "positionX", + "yposition" => "positionY", + "zposition" => "positionZ", + "effect" | "effects" => "effects", + "mask" | "masks" => "masks", + "content" | "contents" => "contents", + "maskshape" | "maskpath" => "path", + "maskfeather" => "feather", + "maskopacity" => "opacity", + "maskexpansion" | "maskoffset" => "expansion", + "sourcetext" => "sourceText", + "timeremap" => "timeRemap", + "audiolevels" => "levels", + _ => return None, + }) +} + +fn child(s: &Session, a: &J) -> R { + let (_, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").unwrap_or(0); + let g: &PropGroup = if uid == 0 || uid == l.props.uid { &l.props } else { l.props.find_group(uid).ok_or("not a property group")? }; + let key = a.get("key").cloned().unwrap_or(J::Null); + let found = match &key { + J::Number(n) => n.as_u64().and_then(|i| g.children.get((i as usize).checked_sub(1)?)), + J::String(k) => { + let chain_of = |n: &Node| -> String { + let mut ch = if uid == 0 || uid == l.props.uid { vec![] } else { chain(l, uid).unwrap_or_default() }; + ch.push(n); + matchnames::node(&ch) + }; + let lk = loose(k); + g.children + .iter() + .find(|n| chain_of(n) == *k) + .or_else(|| g.children.iter().find(|n| n.name() == k)) + .or_else(|| g.children.iter().find(|n| n.match_id() == k)) + .or_else(|| g.children.iter().find(|n| n.name().eq_ignore_ascii_case(k))) + .or_else(|| g.children.iter().find(|n| loose(n.name()) == lk || loose(n.match_id()) == lk)) + .or_else(|| alias(&lk).and_then(|m| g.children.iter().find(|n| n.match_id() == m))) + } + _ => None, + }; + match found { + Some(n) => node_json(c, l, n.uid()), + None => Ok(J::Null), + } +} + +/// A value as the object model shows it: popups 1-based, checkboxes 0/1, layer references as +/// layer indices, paths as `{vertices, inTangents, outTangents, closed}`, text as the document. +fn js_value(c: &Comp, v: &KV) -> J { + match v { + KV::Bool(b) => json!(if *b { 1 } else { 0 }), + KV::Enum(i) => json!(*i + 1), + KV::Layer(l) => json!(l.and_then(|id| c.index_of(LayerId(id))).unwrap_or(0)), + KV::Path(p) => json!({"vertices": p.vertices, "inTangents": p.in_tangents, "outTangents": p.out_tangents, "closed": p.closed}), + KV::Text(d) => text_json(d), + v => v.to_json(), + } +} + +/// A text document for `TextDocument`: the serialized document plus the first character's style. +fn text_json(d: &TextDoc) -> J { + let mut doc = serde_json::to_value(d).unwrap_or(J::Null); + let st = d.style_at(0); + if let Some(o) = doc.as_object_mut() { + o.insert("first".into(), effectcraft_keyframe::text_doc::char_style_json(&st)); + o.insert("para".into(), effectcraft_keyframe::text_doc::para_style_json(&d.para(0))); + } + doc +} + +fn value(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists (it was deleted)")?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let pre = a.get("pre").and_then(J::as_bool).unwrap_or(false); + let v = if pre || !p.has_expression() { + p.value_at(l.layer_time(t)) + } else { + let mut ctx = EvalCtx::new(&s.project, cid, c, t); + ctx.expr = s.expr.as_deref(); + ctx.value(l, p) + }; + Ok(js_value(c, &v)) +} + +fn interp(i: Interp) -> &'static str { + match i { + Interp::Linear => "LINEAR", + Interp::Bezier => "BEZIER", + Interp::Hold => "HOLD", + } +} + +fn keys(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists (it was deleted)")?; + let sel: Vec = s.state.selected_keys.iter().filter(|k| k.layer == l.id && k.prop == uid).map(|k| k.time).collect(); + let _ = cid; + let ease = |e: &[effectcraft_keyframe::Ease], n: usize| -> J { + let n = n.max(1); + // Influence is stored as a fraction; scripts see percent. + let mut v: Vec = e.iter().map(|x| json!({"speed": x.speed, "influence": x.influence * 100.0})).collect(); + let d = e.first().cloned().unwrap_or_default(); + while v.len() < n { + v.push(json!({"speed": d.speed, "influence": d.influence * 100.0})); + } + J::Array(v) + }; + let dims = if p.spatial { 1 } else { p.value.dims().max(1) }; + let out: Vec = p + .keys + .iter() + .map(|k| { + json!({ + "time": l.comp_time(k.time).seconds(), + "layerTime": k.time.seconds(), + "value": js_value(c, &k.value), + "inInterp": interp(k.in_interp), "outInterp": interp(k.out_interp), + "inEase": ease(&k.in_ease, dims), "outEase": ease(&k.out_ease, dims), + "continuous": k.continuous, "autoBezier": k.auto_bezier, "roving": k.roving, + "spatialIn": k.spatial_in, "spatialOut": k.spatial_out, + "spatialContinuous": k.spatial_continuous, "spatialAutoBezier": k.spatial_auto, + "selected": sel.contains(&k.time), + }) + }) + .collect(); + Ok(J::Array(out)) +} + +/// Apply `layer.setText` attribute keys to the Source Text document at `time` and return the new +/// document (for `prop.set` / `prop.addKey`). +fn textdoc(s: &Session, a: &J) -> R { + let (cid, _, l) = layer_of(s, a)?; + let uid = u(a, "uid").ok_or("missing uid")?; + let p = l.props.find(uid).ok_or("the property no longer exists")?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let KV::Text(doc) = p.value_at(l.layer_time(t)) else { return Err("not a Source Text property".into()) }; + let mut d = *doc; + if let Some(attrs) = a.get("attrs").and_then(J::as_object) { + if let Some(text) = attrs.get("text").and_then(J::as_str) { + d.set_text(text); + } + for (k, v) in attrs { + if k == "text" || v.is_null() { + continue; + } + match d.set_attr(k, v, None) { + Ok(true) => {} + Ok(false) => return Err(format!("TextDocument: unknown attribute `{k}`")), + Err(e) => return Err(format!("TextDocument.{k}: {e}")), + } + } + } + serde_json::to_value(&d).map_err(|e| e.to_string()) +} + +fn markers(s: &Session, a: &J) -> R { + let (_, c) = comp_of(s, a)?; + let (list, to_comp): (&[effectcraft_project::Marker], Option<&Layer>) = match u(a, "layer") { + Some(lid) => { + let l = c.layer(LayerId(lid)).ok_or("the layer no longer exists")?; + (&l.markers, Some(l)) + } + None => (&c.markers, None), + }; + let out: Vec = list + .iter() + .map(|m| { + let t = to_comp.map(|l| l.comp_time(m.time)).unwrap_or(m.time); + json!({ + "time": t.seconds(), "duration": m.duration.seconds(), "comment": m.comment, "chapter": m.chapter, "url": m.url, + "frameTarget": m.frame_target, "label": label_index(m.label), "protectedRegion": m.protected, + "cuePointName": m.cue_point.as_ref().map(|c| c.name.clone()).unwrap_or_default(), + "eventCuePoint": m.cue_point.as_ref().is_some_and(|c| !c.navigation), + "params": m.cue_point.as_ref().map(|c| c.params.clone()).unwrap_or_default(), + }) + }) + .collect(); + Ok(J::Array(out)) +} + +fn source_rect(s: &Session, a: &J) -> R { + let (cid, c, l) = layer_of(s, a)?; + let t = f(a, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let mut ctx = EvalCtx::new(&s.project, cid, c, t); + ctx.expr = s.expr.as_deref(); + let r = match &l.source { + LayerSource::Text => { + let paths: Vec<_> = effectcraft_render::text::glyph_paths(&ctx, l).into_iter().map(|(p, _)| p).collect(); + effectcraft_path::bounds(&paths).map(|b| [b.y0, b.x0, b.width(), b.height()]).unwrap_or([0.0; 4]) + } + LayerSource::Shape => match l.props.sub("contents") { + Some(contents) => { + let buf = effectcraft_render::shapes::render(&ctx, l, contents, 1.0); + if buf.img.width <= 4 { [0.0; 4] } else { [2.0 - buf.offset[1], 2.0 - buf.offset[0], buf.img.width as f64 - 4.0, buf.img.height as f64 - 4.0] } + } + None => [0.0; 4], + }, + _ => { + let (w, h) = effectcraft_render::source_size(&s.project, l); + [0.0, 0.0, w as f64, h as f64] + } + }; + Ok(json!({"top": r[0], "left": r[1], "width": r[2], "height": r[3]})) +} + +/// What `PropertyGroup.addProperty(name)` adds to the group `uid`. +fn resolve_add(s: &Session, a: &J) -> R { + let (_, _, l) = layer_of(s, a)?; + let uid = u(a, "uid").unwrap_or(0); + let name = a.get("name").and_then(J::as_str).unwrap_or(""); + let g: &PropGroup = if uid == 0 || uid == l.props.uid { &l.props } else { l.props.find_group(uid).ok_or("not a property group")? }; + let in_contents = g.match_id == "contents"; + let in_group = l.props.sub("contents").is_some_and(|c| c.uid == g.uid || c.find_group(g.uid).is_some()); + Ok(match g.match_id.as_str() { + "effects" => { + let id = matchnames::effect_from_match(name).map(str::to_string).or_else(|| effectcraft_engine::effects::lookup(name).map(|e| e.id.to_string())); + match id { + Some(id) => json!({"kind": "effect", "effect": id}), + None => json!({"error": format!("Can not add a property with name \"{name}\" to this PropertyGroup (unknown effect)")}), + } + } + "masks" => { + if name == "ADBE Mask Atom" || loose(name) == "mask" { + json!({"kind": "mask"}) + } else { + json!({"error": format!("Can not add \"{name}\" to Masks (use \"ADBE Mask Atom\")")}) + } + } + "animators" => json!({"kind": "animator"}), + _ if in_contents && in_group => match matchnames::shape_kind(name) { + Some(k) => { + json!({"kind": "shape", "shape": k, "group": if l.props.sub("contents").is_some_and(|c| c.uid == g.uid) { J::Null } else { json!(g.uid) }}) + } + None => json!({"error": format!("Can not add \"{name}\" to Contents")}), + }, + "properties" if l.props.group("text/animators").is_some_and(|a| a.find_group(g.uid).is_some()) => { + let animator = l.props.group("text/animators").and_then(|a| a.groups().find(|x| x.find_group(g.uid).is_some()).map(|x| x.uid)); + match text_animator_property(name) { + Some(k) => json!({"kind": "animatorProperty", "property": k, "animator": animator}), + None => json!({"error": format!("Can not add \"{name}\" to an animator's Properties")}), + } + } + "selectors" if l.props.group("text/animators").is_some_and(|a| a.find_group(g.uid).is_some()) => { + let animator = l.props.group("text/animators").and_then(|a| a.groups().find(|x| x.find_group(g.uid).is_some()).map(|x| x.uid)); + let kind = match name { + "ADBE Text Wiggly Selector" => "wiggly", + "ADBE Text Expressible Selector" => "expression", + n if loose(n).contains("wiggly") => "wiggly", + n if loose(n).contains("expression") => "expression", + _ => "range", + }; + json!({"kind": "selector", "selector": kind, "animator": animator}) + } + _ => json!({"error": format!("Can not add a property to \"{}\"", g.name)}), + }) +} + +/// A text animator property (`layer.addTextAnimatorProperty` key) from an After Effects match +/// name or a display name. +fn text_animator_property(name: &str) -> Option<&'static str> { + const AE: &[(&str, &str)] = &[ + ("ADBE Text Anchor Point 3D", "anchor"), + ("ADBE Text Position 3D", "position"), + ("ADBE Text Scale 3D", "scale"), + ("ADBE Text Skew", "skew"), + ("ADBE Text Rotation", "rotation"), + ("ADBE Text Opacity", "opacity"), + ("ADBE Text Fill Color", "fillColor"), + ("ADBE Text Fill Hue", "fillHue"), + ("ADBE Text Fill Saturation", "fillSaturation"), + ("ADBE Text Fill Brightness", "fillBrightness"), + ("ADBE Text Fill Opacity", "fillOpacity"), + ("ADBE Text Stroke Color", "strokeColor"), + ("ADBE Text Stroke Hue", "strokeHue"), + ("ADBE Text Stroke Saturation", "strokeSaturation"), + ("ADBE Text Stroke Brightness", "strokeBrightness"), + ("ADBE Text Stroke Opacity", "strokeOpacity"), + ("ADBE Text Stroke Width", "strokeWidth"), + ("ADBE Text Tracking Amount", "tracking"), + ("ADBE Text Line Anchor", "lineAnchor"), + ("ADBE Text Line Spacing", "lineSpacing"), + ("ADBE Text Character Offset", "characterOffset"), + ("ADBE Text Character Replace", "characterValue"), + ("ADBE Text Blur", "blur"), + ]; + if let Some((_, k)) = AE.iter().find(|(m, _)| m.eq_ignore_ascii_case(name)) { + return Some(k); + } + let n = loose(name); + AE.iter().map(|(_, k)| *k).find(|k| loose(k) == n || (n == "anchorpoint" && *k == "anchor") || (n == "trackingamount" && *k == "tracking")) +} + +/// `app.findMenuCommandId(name)`: 1-based index of the command whose label matches (menu items +/// first), 0 when none does. +fn menu_command_id(name: &str) -> J { + let norm = |s: &str| s.trim().trim_end_matches("...").trim_end_matches('…').trim().to_ascii_lowercase(); + let n = norm(name); + let specs = effectcraft_engine::command_specs(); + let pick = specs + .iter() + .position(|c| !c.menu.is_empty() && norm(c.label) == n) + .or_else(|| specs.iter().position(|c| norm(c.label) == n)) + .or_else(|| specs.iter().position(|c| c.id == name)); + json!(pick.map(|i| i + 1).unwrap_or(0)) +} diff --git a/crates/script/src/prelude.js b/crates/script/src/prelude.js new file mode 100644 index 000000000..c4f256f4b --- /dev/null +++ b/crates/script/src/prelude.js @@ -0,0 +1,1690 @@ +// EffectCraft scripting: an After Effects-style object model over the engine. +// +// Written from the behaviour the After Effects Scripting Guide documents (object names, +// attributes, methods). Reads go through `__query(kind, json)`; every edit runs an engine command +// through `__exec(id, json)`, so it is undoable and identical to the UI's action. + +var __ecGlobal = this; + +function __call(id, p) { + var r = __exec(id, JSON.stringify(p || {})); + return r === undefined || r === "" ? null : JSON.parse(r); +} +function __get(kind, a) { + return JSON.parse(__query(kind, JSON.stringify(a || {}))); +} +function __err(msg) { + return new Error(msg); +} +function __num(v, what) { + var n = Number(v); + if (typeof v === "boolean" || isNaN(n)) throw __err((what || "value") + " must be a number"); + return n; +} +function __str(v) { + if (v === undefined || v === null) return ""; + if (v instanceof File || v instanceof Folder) return v.fsName; + return String(v); +} + +// ------------------------------------------------------------------ enums + +function __enum(name, keys, base) { + var e = {}; + for (var i = 0; i < keys.length; i++) e[keys[i]] = base + i; + e.__name = name; + e.__keys = keys; + e.__base = base; + return e; +} +function __enumKey(e, v) { + var i = v - e.__base; + return i >= 0 && i < e.__keys.length ? e.__keys[i] : null; +} + +var BlendingMode = __enum("BlendingMode", [ + "NORMAL", "DISSOLVE", "DANCING_DISSOLVE", "DARKEN", "MULTIPLY", "COLOR_BURN", "CLASSIC_COLOR_BURN", "LINEAR_BURN", + "DARKER_COLOR", "ADD", "LIGHTEN", "SCREEN", "COLOR_DODGE", "CLASSIC_COLOR_DODGE", "LINEAR_DODGE", "LIGHTER_COLOR", + "OVERLAY", "SOFT_LIGHT", "HARD_LIGHT", "LINEAR_LIGHT", "VIVID_LIGHT", "PIN_LIGHT", "HARD_MIX", "DIFFERENCE", + "CLASSIC_DIFFERENCE", "EXCLUSION", "SUBTRACT", "DIVIDE", "HUE", "SATURATION", "COLOR", "LUMINOSITY", "STENCIL_ALPHA", + "STENCIL_LUMA", "SILHOUETE_ALPHA", "SILHOUETTE_LUMA", "ALPHA_ADD", "LUMINESCENT_PREMUL"], 5212); +BlendingMode.SILHOUETTE_ALPHA = BlendingMode.SILHOUETE_ALPHA; +var KeyframeInterpolationType = __enum("KeyframeInterpolationType", ["LINEAR", "BEZIER", "HOLD"], 6612); +var TrackMatteType = __enum("TrackMatteType", ["NO_TRACK_MATTE", "ALPHA", "ALPHA_INVERTED", "LUMA", "LUMA_INVERTED"], 5012); +var LightType = __enum("LightType", ["PARALLEL", "SPOT", "POINT", "AMBIENT", "ENVIRONMENT"], 4412); +var ParagraphJustification = __enum("ParagraphJustification", [ + "LEFT_JUSTIFY", "CENTER_JUSTIFY", "RIGHT_JUSTIFY", "FULL_JUSTIFY_LASTLINE_LEFT", "FULL_JUSTIFY_LASTLINE_CENTER", + "FULL_JUSTIFY_LASTLINE_RIGHT", "FULL_JUSTIFY_LASTLINE_FULL", "MULTIPLE_JUSTIFICATIONS"], 7413); +var PropertyValueType = __enum("PropertyValueType", [ + "NO_VALUE", "ThreeD_SPATIAL", "ThreeD", "TwoD_SPATIAL", "TwoD", "OneD", "COLOR", "CUSTOM_VALUE", "MARKER", + "LAYER_INDEX", "MASK_INDEX", "SHAPE", "TEXT_DOCUMENT"], 6412); +var PropertyType = __enum("PropertyType", ["PROPERTY", "INDEXED_GROUP", "NAMED_GROUP"], 6212); +var LayerQuality = __enum("LayerQuality", ["BEST", "DRAFT", "WIREFRAME"], 4612); +var LayerSamplingQuality = __enum("LayerSamplingQuality", ["BILINEAR", "BICUBIC"], 4812); +var FrameBlendingType = __enum("FrameBlendingType", ["NO_FRAME_BLEND", "FRAME_MIX", "PIXEL_MOTION"], 4012); +var AutoOrientType = __enum("AutoOrientType", ["NO_AUTO_ORIENT", "ALONG_PATH", "CAMERA_OR_POINT_OF_INTEREST", "CHARACTERS_TOWARD_CAMERA"], 4212); +var MaskMode = __enum("MaskMode", ["NONE", "ADD", "SUBTRACT", "INTERSECT", "LIGHTEN", "DARKEN", "DIFFERENCE"], 6812); +var RQItemStatus = __enum("RQItemStatus", ["WILL_CONTINUE", "NEEDS_OUTPUT", "UNQUEUED", "QUEUED", "RENDERING", "USER_STOPPED", "ERR_STOPPED", "DONE"], 3012); +var CloseOptions = __enum("CloseOptions", ["DO_NOT_SAVE_CHANGES", "PROMPT_TO_SAVE_CHANGES", "SAVE_CHANGES"], 1212); +var ImportAsType = __enum("ImportAsType", ["COMP_CROPPED_LAYERS", "FOOTAGE", "COMP", "PROJECT"], 3812); +var TimeDisplayType = __enum("TimeDisplayType", ["FRAMES", "TIMECODE"], 2012); +var LogType = __enum("LogType", ["ERRORS_ONLY", "ERRORS_AND_SETTINGS", "ERRORS_AND_PER_FRAME_INFO"], 3212); +var KeyframeLabel = __enum("KeyframeLabel", ["NONE"], 0); + +var __blendNames = {}; +(function () { + var labels = ["Normal", "Dissolve", "Dancing Dissolve", "Darken", "Multiply", "Color Burn", "Classic Color Burn", + "Linear Burn", "Darker Color", "Add", "Lighten", "Screen", "Color Dodge", "Classic Color Dodge", "Linear Dodge", + "Lighter Color", "Overlay", "Soft Light", "Hard Light", "Linear Light", "Vivid Light", "Pin Light", "Hard Mix", + "Difference", "Classic Difference", "Exclusion", "Subtract", "Divide", "Hue", "Saturation", "Color", "Luminosity", + "Stencil Alpha", "Stencil Luma", "Silhouette Alpha", "Silhouette Luma", "Alpha Add", "Luminescent Premul"]; + for (var i = 0; i < labels.length; i++) __blendNames[labels[i]] = BlendingMode.__base + i; +})(); +function __blendLabel(v) { + for (var k in __blendNames) if (__blendNames[k] === v) return k; + throw __err("Bad blendingMode value " + v); +} + +var __interpKey = { 6612: "linear", 6613: "bezier", 6614: "hold" }; +var __matteKind = { 5013: "alpha", 5014: "alphaInverted", 5015: "luma", 5016: "lumaInverted" }; +var __matteEnum = { Alpha: 5013, AlphaInverted: 5014, Luma: 5015, LumaInverted: 5016 }; +var __justKey = { 7413: "left", 7414: "center", 7415: "right", 7416: "justifyLeft", 7417: "justifyCenter", 7418: "justifyRight", 7419: "justifyAll" }; + +// ------------------------------------------------------------------ output, $ and dialogs + +function writeLn(s) { __print(__str(s)); } +function write(s) { __print(__str(s)); } +function alert(s) { __print(__str(s)); } +function confirm(s) { __print(__str(s)); return true; } +function prompt(msg, def) { __print(__str(msg)); return def === undefined ? "" : def; } +function clearOutput() {} + +var $ = { + writeln: function () { __print(Array.prototype.slice.call(arguments).map(__str).join("")); }, + write: function () { __print(Array.prototype.slice.call(arguments).map(__str).join("")); }, + sleep: function () {}, + gc: function () {}, + global: __ecGlobal, + level: 0, + engineName: "boa", + get fileName() { return __get("app").scriptName; }, + get os() { return "EffectCraft"; }, + get version() { return "EffectCraft scripting"; }, + get locale() { return "en_US"; }, + evalFile: function () { throw __err("$.evalFile is not supported: scripts can't run scripts"); }, +}; + +// ------------------------------------------------------------------ File / Folder + +function __join(a, b) { return a.replace(/[\/\\]+$/, "") + "/" + b; } + +function File(path) { + if (!(this instanceof File)) return new File(path); + this.__path = path === undefined ? "" : __str(path); + this.__buf = null; + this.__mode = null; + this.__pos = 0; + this.encoding = "UTF-8"; + this.lineFeed = "Unix"; + this.error = ""; +} +File.prototype = { + constructor: File, + get fsName() { return this.__path; }, + get fullName() { return this.__path; }, + get absoluteURI() { return this.__path; }, + get name() { return this.__path.split(/[\/\\]/).pop(); }, + get displayName() { return this.name; }, + get path() { var p = this.__path.split(/[\/\\]/); p.pop(); return p.join("/"); }, + get parent() { return new Folder(this.path); }, + get exists() { return JSON.parse(__file("exists", this.__path)) === true; }, + get length() { return this.__buf !== null ? this.__buf.length : (this.exists ? JSON.parse(__file("read", this.__path)).length : 0); }, + get eof() { return this.__buf === null || this.__pos >= this.__buf.length; }, + toString: function () { return this.__path; }, + open: function (mode) { + mode = (mode || "r").charAt(0).toLowerCase(); + this.__mode = mode; + this.__pos = 0; + if (mode === "r" || mode === "e") { + // Throws when the location is off limits (the scripting file gate). + if (!JSON.parse(__file("access", this.__path))) { + if (mode === "r") { this.error = "File not found"; return false; } + this.__buf = ""; + return true; + } + this.__buf = JSON.parse(__file("read", this.__path)); + } else if (mode === "a") { + this.__buf = this.exists ? JSON.parse(__file("read", this.__path)) : ""; + this.__pos = this.__buf.length; + } else { + // Check the gate now, like After Effects refusing to open a file for writing. + JSON.parse(__file("write", this.__path, "")); + this.__buf = ""; + } + return true; + }, + read: function (n) { + if (this.__buf === null) this.open("r"); + if (this.__buf === null) return ""; + var s = n === undefined ? this.__buf.substring(this.__pos) : this.__buf.substr(this.__pos, n); + this.__pos += s.length; + return s; + }, + readln: function () { + if (this.__buf === null) this.open("r"); + if (this.__buf === null) return ""; + var i = this.__buf.indexOf("\n", this.__pos); + var end = i < 0 ? this.__buf.length : i; + var s = this.__buf.substring(this.__pos, end).replace(/\r$/, ""); + this.__pos = i < 0 ? this.__buf.length : i + 1; + return s; + }, + write: function () { + if (this.__mode !== "w" && this.__mode !== "a" && this.__mode !== "e") throw __err("File is not open for writing"); + var s = Array.prototype.slice.call(arguments).map(__str).join(""); + this.__buf = this.__buf.substring(0, this.__pos) + s + this.__buf.substring(this.__pos + s.length); + this.__pos += s.length; + return true; + }, + writeln: function () { return this.write(Array.prototype.slice.call(arguments).map(__str).join("") + "\n"); }, + seek: function (p) { this.__pos = p; return true; }, + tell: function () { return this.__pos; }, + close: function () { + if ((this.__mode === "w" || this.__mode === "a" || this.__mode === "e") && this.__buf !== null) { + __file("write", this.__path, this.__buf); + } + this.__buf = null; + this.__mode = null; + return true; + }, + remove: function () { return JSON.parse(__file("remove", this.__path)); }, + copy: function (to) { + var data = JSON.parse(__file("read", this.__path)); + __file("write", __str(to), data); + return true; + }, + rename: function () { throw __err("File.rename is not supported"); }, + execute: function () { throw __err("scripts can't execute files"); }, + openDlg: function () { return null; }, + saveDlg: function () { return null; }, +}; +File.openDialog = function () { return null; }; +File.saveDialog = function () { return null; }; +File.decode = function (s) { return decodeURIComponent(s); }; +File.encode = function (s) { return encodeURIComponent(s); }; +File.fs = "EffectCraft"; + +function Folder(path) { + if (!(this instanceof Folder)) return new Folder(path); + this.__path = path === undefined ? "" : __str(path); +} +Folder.prototype = { + constructor: Folder, + get fsName() { return this.__path; }, + get fullName() { return this.__path; }, + get absoluteURI() { return this.__path; }, + get name() { return this.__path.replace(/[\/\\]+$/, "").split(/[\/\\]/).pop(); }, + get displayName() { return this.name; }, + get path() { var p = this.__path.replace(/[\/\\]+$/, "").split(/[\/\\]/); p.pop(); return p.join("/"); }, + get parent() { return new Folder(this.path); }, + get exists() { return JSON.parse(__file("isDir", this.__path)) === true; }, + toString: function () { return this.__path; }, + create: function () { return JSON.parse(__file("mkdir", this.__path)); }, + remove: function () { return JSON.parse(__file("remove", this.__path)); }, + getFiles: function (mask) { + var names = JSON.parse(__file("list", this.__path)); + var out = []; + for (var i = 0; i < names.length; i++) { + var n = names[i]; + if (typeof mask === "string" && mask !== "*") { + var re = new RegExp("^" + mask.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".") + "$", "i"); + if (!re.test(n.split(/[\/\\]/).pop())) continue; + } + var isDir = JSON.parse(__file("isDir", n)); + var f = isDir ? new Folder(n) : new File(n); + if (typeof mask === "function" && !mask(f)) continue; + out.push(f); + } + return out; + }, + selectDlg: function () { return null; }, +}; +Folder.selectDialog = function () { return null; }; +Object.defineProperty(Folder, "current", { get: function () { var d = __get("app").projectDir; return d ? new Folder(d) : null; } }); +Object.defineProperty(Folder, "temp", { get: function () { return null; } }); +Object.defineProperty(Folder, "desktop", { get: function () { return null; } }); +Object.defineProperty(Folder, "myDocuments", { get: function () { return null; } }); + +function Socket() { __file("network", ""); } +var system = { callSystem: function () { throw __err("scripts can't run system commands"); } }; + +// ------------------------------------------------------------------ helpers + +// Unknown attributes of property groups and layers resolve to child properties, like After +// Effects (`layer.transform.position`, `layer.Effects`, `effect.blurriness`). +var __attr = { + get: function (t, k, r) { + if (typeof k !== "string" || k in t || k.charAt(0) === "_" || k === "toJSON" || k === "then") return Reflect.get(t, k, r); + var c = t.property(k); + return c === null ? undefined : c; + }, +}; + +// 1-based collections with `[n]` access. +var __coll = { + get: function (t, k, r) { + if (typeof k === "string" && /^[0-9]+$/.test(k)) return t.__at(parseInt(k, 10)); + return Reflect.get(t, k, r); + }, +}; + +function KeyframeEase(speed, influence) { + if (!(this instanceof KeyframeEase)) return new KeyframeEase(speed, influence); + this.speed = speed === undefined ? 0 : speed; + this.influence = influence === undefined ? 16.666666667 : influence; +} + +function Shape() { + if (!(this instanceof Shape)) return new Shape(); + this.vertices = []; + this.inTangents = []; + this.outTangents = []; + this.closed = true; + this.featherSegLocs = []; + this.featherRelSegLocs = []; + this.featherRadii = []; + this.featherInterps = []; + this.featherTensions = []; + this.featherTypes = []; + this.featherRelCornerAngles = []; +} + +function MarkerValue(comment, chapter, url, frameTarget, cuePointName, params) { + if (!(this instanceof MarkerValue)) return new MarkerValue(comment, chapter, url, frameTarget, cuePointName, params); + this.comment = comment === undefined ? "" : String(comment); + this.chapter = chapter === undefined ? "" : String(chapter); + this.url = url === undefined ? "" : String(url); + this.frameTarget = frameTarget === undefined ? "" : String(frameTarget); + this.cuePointName = cuePointName === undefined ? "" : String(cuePointName); + this.duration = 0; + this.eventCuePoint = false; + this.label = 0; + this.protectedRegion = false; + this.__params = []; + if (params) this.setParameters(params); +} +MarkerValue.prototype.getParameters = function () { + var o = {}; + for (var i = 0; i < this.__params.length; i++) o[this.__params[i][0]] = this.__params[i][1]; + return o; +}; +MarkerValue.prototype.setParameters = function (o) { + this.__params = []; + for (var k in o) this.__params.push([k, String(o[k])]); +}; + +// ------------------------------------------------------------------ TextDocument + +// `layer.setText` keys of each TextDocument attribute. +var __textKeys = { + font: "font", fontStyle: "style", fontSize: "size", fillColor: "fill", strokeColor: "stroke", strokeWidth: "strokeWidth", + applyFill: "applyFill", applyStroke: "applyStroke", strokeOverFill: "strokeOverFill", tracking: "tracking", + baselineShift: "baselineShift", horizontalScale: "hScale", verticalScale: "vScale", fauxBold: "fauxBold", + fauxItalic: "fauxItalic", allCaps: "allCaps", smallCaps: "smallCaps", tsume: "tsume", superscript: "superscript", + subscript: "subscript", ligature: "ligatures", startIndent: "indentLeft", endIndent: "indentRight", + firstLineIndent: "indentFirst", spaceBefore: "spaceBefore", spaceAfter: "spaceAfter", +}; + +function TextDocument(text) { + if (!(this instanceof TextDocument)) return new TextDocument(text); + this.__base = null; + this.__changed = {}; + this.__text = text === undefined ? "" : String(text); + this.__textSet = text !== undefined; +} +TextDocument.__from = function (doc) { + var t = new TextDocument(); + t.__base = doc; + t.__text = doc.text; + t.__textSet = false; + return t; +}; +TextDocument.prototype = { + constructor: TextDocument, + get text() { return this.__text; }, + set text(v) { this.__text = String(v); this.__textSet = true; }, + __style: function (k, d) { + if (k in this.__changed) return this.__changed[k]; + if (this.__base && this.__base.first && k in this.__base.first) return this.__base.first[k]; + if (this.__base && this.__base.para && k in this.__base.para) return this.__base.para[k]; + return d; + }, + __set: function (k, v) { this.__changed[k] = v; }, + get font() { return this.__style("font", "Inter"); }, + set font(v) { this.__set("font", String(v)); }, + get fontFamily() { return this.font; }, + get fontStyle() { return this.__style("style", "Regular"); }, + set fontStyle(v) { this.__set("style", String(v)); }, + get fontSize() { return this.__style("size", 72); }, + set fontSize(v) { this.__set("size", __num(v, "fontSize")); }, + get fillColor() { var c = this.__style("fill", [1, 1, 1, 1]); return [c[0], c[1], c[2]]; }, + set fillColor(v) { this.__set("fill", [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]); }, + get strokeColor() { var c = this.__style("stroke", [0, 0, 0, 1]); return [c[0], c[1], c[2]]; }, + set strokeColor(v) { this.__set("stroke", [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]); }, + get strokeWidth() { return this.__style("strokeWidth", 0); }, + set strokeWidth(v) { this.__set("strokeWidth", __num(v, "strokeWidth")); }, + get applyFill() { return this.__style("applyFill", true); }, + set applyFill(v) { this.__set("applyFill", !!v); }, + get applyStroke() { return this.__style("applyStroke", false); }, + set applyStroke(v) { this.__set("applyStroke", !!v); }, + get strokeOverFill() { return "strokeOverFill" in this.__changed ? this.__changed.strokeOverFill : (this.__base ? this.__base.stroke_over_fill : true); }, + set strokeOverFill(v) { this.__set("strokeOverFill", !!v); }, + get tracking() { return this.__style("tracking", 0); }, + set tracking(v) { this.__set("tracking", __num(v, "tracking")); }, + get leading() { var l = this.__style("leading", "auto"); return l === "auto" ? this.fontSize * 1.2 : l; }, + set leading(v) { this.__set("leading", __num(v, "leading")); }, + get autoLeading() { return this.__style("leading", "auto") === "auto"; }, + set autoLeading(v) { if (v) this.__set("leading", "auto"); }, + get justification() { + var j = this.__style("justify", "left"); + for (var k in __justKey) if (__justKey[k] === j) return Number(k); + return ParagraphJustification.LEFT_JUSTIFY; + }, + set justification(v) { + var k = __justKey[v]; + if (!k) throw __err("Bad justification value " + v); + this.__set("justify", k); + }, + get baselineShift() { return this.__style("baselineShift", 0); }, + set baselineShift(v) { this.__set("baselineShift", __num(v)); }, + get horizontalScale() { return this.__style("hScale", 100) / 100; }, + set horizontalScale(v) { this.__set("hScale", __num(v) * 100); }, + get verticalScale() { return this.__style("vScale", 100) / 100; }, + set verticalScale(v) { this.__set("vScale", __num(v) * 100); }, + get fauxBold() { return this.__style("fauxBold", false); }, + set fauxBold(v) { this.__set("fauxBold", !!v); }, + get fauxItalic() { return this.__style("fauxItalic", false); }, + set fauxItalic(v) { this.__set("fauxItalic", !!v); }, + get allCaps() { return this.__style("allCaps", false); }, + set allCaps(v) { this.__set("allCaps", !!v); }, + get smallCaps() { return this.__style("smallCaps", false); }, + set smallCaps(v) { this.__set("smallCaps", !!v); }, + get superscript() { return this.__style("baseline", "normal") === "superscript"; }, + set superscript(v) { this.__set("superscript", !!v); }, + get subscript() { return this.__style("baseline", "normal") === "subscript"; }, + set subscript(v) { this.__set("subscript", !!v); }, + get tsume() { return this.__style("tsume", 0); }, + set tsume(v) { this.__set("tsume", __num(v)); }, + get boxText() { return !!(this.__base && this.__base.box_size); }, + get pointText() { return !this.boxText; }, + get boxTextSize() { return this.__base && this.__base.box_size ? this.__base.box_size.slice() : undefined; }, + set boxTextSize(v) { + var p = this.boxTextPos || [0, 0]; + this.__set("box", [p[0], p[1], v[0], v[1]]); + }, + get boxTextPos() { return this.__base && this.__base.box_size ? this.__base.box_pos.slice() : undefined; }, + get startIndent() { return this.__style("indentLeft", 0); }, + set startIndent(v) { this.__set("indentLeft", __num(v)); }, + get endIndent() { return this.__style("indentRight", 0); }, + set endIndent(v) { this.__set("indentRight", __num(v)); }, + get firstLineIndent() { return this.__style("indentFirst", 0); }, + set firstLineIndent(v) { this.__set("indentFirst", __num(v)); }, + get spaceBefore() { return this.__style("spaceBefore", 0); }, + set spaceBefore(v) { this.__set("spaceBefore", __num(v)); }, + get spaceAfter() { return this.__style("spaceAfter", 0); }, + set spaceAfter(v) { this.__set("spaceAfter", __num(v)); }, + resetCharStyle: function () { + var d = { font: "Inter", style: "Regular", size: 72, fill: [1, 1, 1, 1], stroke: [0, 0, 0, 1], strokeWidth: 0, applyFill: true, + applyStroke: false, tracking: 0, leading: "auto", baselineShift: 0, hScale: 100, vScale: 100, tsume: 0, fauxBold: false, + fauxItalic: false, allCaps: false, smallCaps: false }; + for (var k in d) this.__changed[k] = d[k]; + }, + resetParagraphStyle: function () { + var d = { justify: "left", indentLeft: 0, indentRight: 0, indentFirst: 0, spaceBefore: 0, spaceAfter: 0 }; + for (var k in d) this.__changed[k] = d[k]; + }, + toString: function () { return this.__text; }, + // `layer.setText` attributes to apply. + __attrs: function () { + var a = {}; + for (var k in this.__changed) a[k] = this.__changed[k]; + if (this.__textSet || !this.__base || this.__text !== this.__base.text) a.text = this.__text; + return a; + }, +}; + +// ------------------------------------------------------------------ properties + +function __propOf(layer, info) { + if (!info) return null; + if (info.isLayer) return layer; + if (info.kind === "group") return info.isMask ? new MaskPropertyGroup(layer, info.uid) : new PropertyGroup(layer, info.uid); + return new Property(layer, info.uid); +} + +function PropertyBase() {} +PropertyBase.prototype = { + __info: function () { return __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid }); }, + __ref: function (p) { + p = p || {}; + p.comp = this.__layer.__comp; + p.layer = this.__layer.__id; + p.prop = this.__uid; + return p; + }, + get name() { return this.__info().name; }, + set name(v) { + var i = this.__info(); + if (!i.instance) throw __err("Can not set the name of \"" + i.name + "\": it is not an indexed group member"); + __call("prop.renameGroup", this.__ref({ name: String(v) })); + }, + get matchName() { return this.__info().matchName; }, + get propertyIndex() { return this.__info().index; }, + get propertyDepth() { return this.__info().depth; }, + get propertyType() { return PropertyType[this.__info().propertyType]; }, + get parentProperty() { + var i = this.__info(); + if (i.depth <= 1) return this.__layer; + return __propOf(this.__layer, __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: i.parent })); + }, + get isModified() { return !!this.__info().isModified; }, + get canSetEnabled() { return !!this.__info().canSetEnabled; }, + get enabled() { return this.__info().enabled !== false; }, + set enabled(v) { + if (!this.canSetEnabled) throw __err("Can not set enabled on \"" + this.name + "\""); + __call("prop.setGroupEnabled", this.__ref({ value: !!v })); + }, + get active() { return this.enabled && this.__layer.enabled; }, + get elided() { return false; }, + get isEffect() { return !!this.__info().isEffect; }, + get isMask() { return !!this.__info().isMask; }, + get selected() { + var c = __get("item", { id: this.__layer.__comp }).selectedProperties; + for (var i = 0; i < c.length; i++) if (c[i][0] === this.__layer.__id && c[i][1] === this.__uid) return true; + return false; + }, + set selected(v) { + if (v) __call("prop.select", { comp: this.__layer.__comp, layer: this.__layer.__id, prop: this.__uid, add: true }); + }, + propertyGroup: function (n) { + var p = this; + n = n === undefined ? 1 : n; + for (var i = 0; i < n && p; i++) p = p.parentProperty; + return p || null; + }, + remove: function () { __call("prop.removeGroup", this.__ref()); }, + moveTo: function (i) { __call("prop.moveGroup", this.__ref({ index: __num(i, "index") })); }, + duplicate: function () { + var r = __call("prop.duplicateGroup", this.__ref()); + return __propOf(this.__layer, __get("node", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: r.prop })); + }, + toString: function () { return "[object " + this.__kind + "]"; }, +}; + +function PropertyGroup(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "PropertyGroup"; + return new Proxy(this, __attr); +} +PropertyGroup.prototype = Object.create(PropertyBase.prototype); +PropertyGroup.prototype.constructor = PropertyGroup; +Object.defineProperty(PropertyGroup.prototype, "numProperties", { get: function () { return this.__info().numProperties; } }); +PropertyGroup.prototype.property = function (key) { + if (key === undefined || key === null) throw __err("property(): missing name or index"); + var k = typeof key === "number" ? key : String(key); + return __propOf(this.__layer, __get("child", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, key: k })); +}; +PropertyGroup.prototype.canAddProperty = function (name) { + var r = __get("resolveAdd", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, name: String(name) }); + return !r.error; +}; +PropertyGroup.prototype.addProperty = function (name) { + var L = this.__layer; + var r = __get("resolveAdd", { comp: L.__comp, layer: L.__id, uid: this.__uid, name: String(name) }); + if (r.error) throw __err(r.error); + var uid; + if (r.kind === "effect") uid = __call("effect.apply", { comp: L.__comp, layers: [L.__id], effect: r.effect }).effects[0]; + else if (r.kind === "mask") uid = __call("layer.addMask", { comp: L.__comp, layer: L.__id }).mask; + else if (r.kind === "shape") { + var p = { comp: L.__comp, layer: L.__id, kind: r.shape }; + if (r.group !== null && r.group !== undefined) p.group = r.group; + uid = __call("layer.addShapeItem", p).uid; + } else if (r.kind === "animator") { + uid = __call("layer.addTextAnimator", { comp: L.__comp, layer: L.__id, properties: [] }).animator; + } else if (r.kind === "animatorProperty") { + var before = this.numProperties; + __call("layer.addTextAnimatorProperty", { comp: L.__comp, layer: L.__id, animator: r.animator, property: r.property }); + return this.property(before + 1); + } else if (r.kind === "selector") { + var n0 = this.numProperties; + __call("layer.addTextSelector", { comp: L.__comp, layer: L.__id, animator: r.animator, kind: r.selector }); + return this.property(n0 + 1); + } else throw __err("Can not add \"" + name + "\" here"); + return __propOf(L, __get("node", { comp: L.__comp, layer: L.__id, uid: uid })); +}; + +function MaskPropertyGroup(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "MaskPropertyGroup"; + return new Proxy(this, __attr); +} +MaskPropertyGroup.prototype = Object.create(PropertyGroup.prototype); +MaskPropertyGroup.prototype.constructor = MaskPropertyGroup; +(function () { + var modes = { "None": 6812, "Add": 6813, "Subtract": 6814, "Intersect": 6815, "Lighten": 6816, "Darken": 6817, "Difference": 6818 }; + function mask(o) { return { comp: o.__layer.__comp, layer: o.__layer.__id, mask: o.__uid }; } + Object.defineProperty(MaskPropertyGroup.prototype, "maskMode", { + get: function () { return modes[this.__info().maskMode]; }, + set: function (v) { + var p = mask(this); + for (var k in modes) if (modes[k] === v) p.mode = k; + if (!p.mode) throw __err("Bad MaskMode value " + v); + __call("layer.mask.mode", p); + }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "inverted", { + get: function () { return !!this.__info().inverted; }, + set: function (v) { var p = mask(this); p.value = !!v; __call("layer.mask.invert", p); }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "locked", { + get: function () { return !!this.__info().locked; }, + set: function (v) { var p = mask(this); p.value = !!v; __call("layer.mask.lock", p); }, + }); + Object.defineProperty(MaskPropertyGroup.prototype, "color", { get: function () { return this.__info().color; } }); + MaskPropertyGroup.prototype.remove = function () { __call("layer.mask.remove", mask(this)); }; +})(); + +function Property(layer, uid) { + this.__layer = layer; + this.__uid = uid; + this.__kind = "Property"; + return new Proxy(this, __attr); +} +Property.prototype = Object.create(PropertyBase.prototype); +Property.prototype.constructor = Property; +Property.prototype.property = function () { return null; }; +(function () { + var P = Property.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + def("propertyValueType", function () { return PropertyValueType[this.__info().valueType]; }); + def("value", function () { return this.valueAtTime(this.__layer.containingComp.time, false); }); + def("numKeys", function () { return this.__info().numKeys; }); + def("expression", function () { return this.__info().expression; }, function (v) { + if (!this.canSetExpression) throw __err("Can not set an expression on \"" + this.name + "\""); + __call("prop.setExpression", this.__ref({ expression: String(v) })); + }); + def("expressionEnabled", function () { return !!this.__info().expressionEnabled; }, function (v) { + __call("prop.setExpression", this.__ref({ enabled: !!v })); + }); + def("expressionError", function () { + var e = this.expression; + return e ? __get("exprError", { code: e }) : ""; + }); + def("canSetExpression", function () { return !!this.__info().canSetExpression; }); + def("canVaryOverTime", function () { return !!this.__info().canVaryOverTime; }); + def("isTimeVarying", function () { var i = this.__info(); return i.numKeys > 0 || !!i.expressionEnabled; }); + def("isSpatial", function () { return !!this.__info().spatial; }); + def("unitsText", function () { return this.__info().ui; }); + def("hasMin", function () { return this.__info().min !== undefined; }); + def("hasMax", function () { return this.__info().max !== undefined; }); + def("minValue", function () { var i = this.__info(); if (i.min === undefined) throw __err("property has no minimum"); return i.min; }); + def("maxValue", function () { var i = this.__info(); if (i.max === undefined) throw __err("property has no maximum"); return i.max; }); + def("dimensionsSeparated", function () { return !!this.__info().separated; }, function (v) { + __call("prop.separateDimensions", { comp: this.__layer.__comp, layer: this.__layer.__id, value: !!v }); + }); + def("isSeparationFollower", function () { return !!this.__info().separationFollower; }); + def("isSeparationLeader", function () { return this.__info().matchId === "position" && this.__info().depth === 2; }); + def("separationDimension", function () { var m = this.__info().matchId; return m === "positionY" ? 1 : m === "positionZ" ? 2 : 0; }); + def("separationLeader", function () { return this.isSeparationFollower ? this.parentProperty.property("ADBE Position") : null; }); + def("selectedKeys", function () { + var k = this.__keys(), out = []; + for (var i = 0; i < k.length; i++) if (k[i].selected) out.push(i + 1); + return out; + }); + def("isDropdownEffect", function () { return false; }); + + P.getSeparationFollower = function (dim) { + return this.parentProperty.property(["ADBE Position_0", "ADBE Position_1", "ADBE Position_2"][dim]); + }; + P.__keys = function () { return __get("keys", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid }); }; + P.__key = function (i) { + var k = this.__keys(); + i = __num(i, "keyIndex"); + if (i < 1 || i > k.length || Math.floor(i) !== i) throw __err("keyIndex " + i + " is out of range 1.." + k.length); + return k[i - 1]; + }; + // Script value → engine JSON for `prop.set` / `prop.addKey`. + P.__toEngine = function (v, info, time) { + var t = info.valueType; + if (t === "TEXT_DOCUMENT") { + var td = v instanceof TextDocument ? v : new TextDocument(String(v)); + return __get("textdoc", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, time: time, attrs: td.__attrs() }); + } + if (t === "SHAPE") { + if (!v || !v.vertices) throw __err("value must be a Shape"); + var n = v.vertices.length; + function fit(a) { var o = []; for (var i = 0; i < n; i++) o.push(a && a[i] ? [a[i][0], a[i][1]] : [0, 0]); return o; } + return { vertices: fit(v.vertices), in_tangents: fit(v.inTangents), out_tangents: fit(v.outTangents), closed: v.closed !== false }; + } + if (t === "LAYER_INDEX") { + var idx = v && v.__id !== undefined ? v.index : __num(v); + if (!idx) return null; + return this.__layer.containingComp.layer(idx).__id; + } + if (info.type === "enum") return __num(v) - 1; + if (info.type === "bool") return typeof v === "boolean" ? v : __num(v) !== 0; + if (info.type === "color") { + if (!(v instanceof Array) || v.length < 3) throw __err("value must be an array of 3 or 4 numbers"); + return [v[0], v[1], v[2], v.length > 3 ? v[3] : 1]; + } + if (info.dims > 1) { + if (!(v instanceof Array)) throw __err("value must be an array of " + info.dims + " numbers"); + var o = []; + for (var d = 0; d < info.dims; d++) o.push(d < v.length ? __num(v[d]) : (d === 2 ? 0 : __num(v[v.length - 1]))); + return o; + } + if (v instanceof Array) throw __err("value must be a number, not an array"); + return __num(v); + }; + P.__fromEngine = function (v, info) { + if (info.valueType === "TEXT_DOCUMENT") return TextDocument.__from(v); + if (info.valueType === "SHAPE") { + var s = new Shape(); + s.vertices = v.vertices; + s.inTangents = v.inTangents; + s.outTangents = v.outTangents; + s.closed = v.closed; + return s; + } + if (info.type === "color" && v instanceof Array) return v.slice(); + return v; + }; + P.valueAtTime = function (t, preExpression) { + var info = this.__info(); + var v = __get("value", { comp: this.__layer.__comp, layer: this.__layer.__id, uid: this.__uid, time: __num(t, "time"), pre: !!preExpression }); + return this.__fromEngine(v, info); + }; + P.setValue = function (v) { + var info = this.__info(); + if (info.numKeys > 0) throw __err("Can not use setValue on \"" + info.name + "\": it has keyframes (use setValueAtTime or setValueAtKey)"); + if (info.hidden) throw __err("\"" + info.name + "\" can not be set"); + __call("prop.set", this.__ref({ value: this.__toEngine(v, info) })); + }; + P.setValueAtTime = function (t, v) { + var info = this.__info(); + t = __num(t, "time"); + __call("prop.addKey", this.__ref({ time: this.__layer.__layerTime(t), value: this.__toEngine(v, info, t) })); + }; + P.setValuesAtTimes = function (ts, vs) { + if (!(ts instanceof Array) || !(vs instanceof Array) || ts.length !== vs.length) throw __err("setValuesAtTimes: times and values must be arrays of the same length"); + for (var i = 0; i < ts.length; i++) this.setValueAtTime(ts[i], vs[i]); + }; + P.setValueAtKey = function (i, v) { + var k = this.__key(i), info = this.__info(); + __call("keys.set", this.__ref({ time: k.layerTime, value: this.__toEngine(v, info, k.time) })); + }; + P.addKey = function (t) { + t = __num(t, "time"); + __call("prop.addKey", this.__ref({ time: this.__layer.__layerTime(t) })); + return this.nearestKeyIndex(t); + }; + P.nearestKeyIndex = function (t) { + var k = this.__keys(), best = 0, d = Infinity; + if (!k.length) throw __err("\"" + this.name + "\" has no keyframes"); + for (var i = 0; i < k.length; i++) { + var e = Math.abs(k[i].time - t); + if (e < d) { d = e; best = i + 1; } + } + return best; + }; + P.keyTime = function (i) { return this.__key(i).time; }; + P.keyValue = function (i) { return this.__fromEngine(this.__key(i).value, this.__info()); }; + // Select key `i` (only) so the selection-based keyframe commands act on it. + P.__select = function (i) { + var k = this.__key(i); + __call("keys.select", { comp: this.__layer.__comp, keys: [{ layer: this.__layer.__id, prop: this.__uid, time: k.layerTime }] }); + return k; + }; + P.removeKey = function (i) { + this.__select(i); + __call("keys.delete", { comp: this.__layer.__comp }); + }; + P.keyInInterpolationType = function (i) { return KeyframeInterpolationType[this.__key(i).inInterp]; }; + P.keyOutInterpolationType = function (i) { return KeyframeInterpolationType[this.__key(i).outInterp]; }; + P.isInterpolationTypeValid = function (t) { return t === 6612 || t === 6613 || t === 6614; }; + P.setInterpolationTypeAtKey = function (i, inType, outType) { + if (outType === undefined) outType = inType; + if (!__interpKey[inType] || !__interpKey[outType]) throw __err("Bad KeyframeInterpolationType"); + this.__select(i); + __call("keys.interpolation", { comp: this.__layer.__comp, "in": __interpKey[inType], out: __interpKey[outType] }); + }; + P.keyInTemporalEase = function (i) { return this.__key(i).inEase.map(function (e) { return new KeyframeEase(e.speed, e.influence); }); }; + P.keyOutTemporalEase = function (i) { return this.__key(i).outEase.map(function (e) { return new KeyframeEase(e.speed, e.influence); }); }; + P.setTemporalEaseAtKey = function (i, inEase, outEase) { + if (outEase === undefined) outEase = inEase; + if (!(inEase instanceof Array) || !(outEase instanceof Array)) throw __err("setTemporalEaseAtKey: eases must be arrays of KeyframeEase"); + this.__select(i); + function sp(a) { return a.map(function (e) { return __num(e.speed, "speed"); }); } + function inf(a) { + return a.map(function (e) { + var x = __num(e.influence, "influence"); + if (x < 0.1 || x > 100) throw __err("influence must be between 0.1 and 100"); + return x; + }); + } + __call("keys.velocity", { comp: this.__layer.__comp, inSpeed: sp(inEase), inInfluence: inf(inEase), outSpeed: sp(outEase), outInfluence: inf(outEase) }); + }; + P.keyTemporalContinuous = function (i) { return !!this.__key(i).continuous; }; + P.setTemporalContinuousAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, continuous: !!b }); }; + P.keyTemporalAutoBezier = function (i) { return !!this.__key(i).autoBezier; }; + P.setTemporalAutoBezierAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, autoBezier: !!b }); }; + P.keyRoving = function (i) { return !!this.__key(i).roving; }; + P.setRovingAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, roving: !!b }); }; + P.keySpatialInTangent = function (i) { return this.__key(i).spatialIn.slice(0, this.__info().dims); }; + P.keySpatialOutTangent = function (i) { return this.__key(i).spatialOut.slice(0, this.__info().dims); }; + P.keySpatialContinuous = function (i) { return !!this.__key(i).spatialContinuous; }; + P.keySpatialAutoBezier = function (i) { return !!this.__key(i).spatialAutoBezier; }; + P.setSpatialTangentsAtKey = function (i, inT, outT) { + var k = this.__key(i); + __call("keys.setSpatialTangents", this.__ref({ time: k.layerTime, "in": inT, out: outT === undefined ? inT : outT })); + }; + P.setSpatialContinuousAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, spatial: b ? "continuousBezier" : "bezier" }); }; + P.setSpatialAutoBezierAtKey = function (i, b) { this.__select(i); __call("keys.interpolation", { comp: this.__layer.__comp, spatial: b ? "autoBezier" : "bezier" }); }; + P.keySelected = function (i) { return !!this.__key(i).selected; }; + P.setSelectedAtKey = function (i, b) { + var k = this.__key(i); + if (b) { + __call("keys.select", { comp: this.__layer.__comp, add: true, keys: [{ layer: this.__layer.__id, prop: this.__uid, time: k.layerTime }] }); + } else { + var keep = [], all = this.__keys(); + for (var j = 0; j < all.length; j++) if (all[j].selected && j !== i - 1) keep.push({ layer: this.__layer.__id, prop: this.__uid, time: all[j].layerTime }); + __call("keys.select", { comp: this.__layer.__comp, keys: keep }); + } + }; + P.keyLabel = function () { return 0; }; + P.setLabelAtKey = function () {}; + P.addToMotionGraphicsTemplate = function () { return false; }; + P.canAddToMotionGraphicsTemplate = function () { return false; }; +})(); + +// Layer / composition markers (`layer.marker`, `comp.markerProperty`): keyframes are markers. +function MarkerProperty(comp, layer) { + this.__compId = comp; + this.__layerRef = layer; + this.__kind = "Property"; +} +MarkerProperty.prototype = { + constructor: MarkerProperty, + __p: function (o) { + o = o || {}; + o.comp = this.__compId; + if (this.__layerRef) o.layer = this.__layerRef.__id; + return o; + }, + __list: function () { return __get("markers", this.__p()); }, + get name() { return "Marker"; }, + get matchName() { return "ADBE Marker"; }, + get propertyValueType() { return PropertyValueType.MARKER; }, + get propertyType() { return PropertyType.PROPERTY; }, + get numKeys() { return this.__list().length; }, + get value() { return null; }, + get canSetExpression() { return false; }, + get canVaryOverTime() { return true; }, + get isTimeVarying() { return this.numKeys > 0; }, + get parentProperty() { return this.__layerRef || null; }, + __at: function (i) { + var l = this.__list(); + if (i < 1 || i > l.length) throw __err("marker index " + i + " is out of range 1.." + l.length); + return l[i - 1]; + }, + keyTime: function (i) { return this.__at(i).time; }, + keyValue: function (i) { + var m = this.__at(i); + var v = new MarkerValue(m.comment, m.chapter, m.url, m.frameTarget, m.cuePointName); + v.duration = m.duration; + v.eventCuePoint = m.eventCuePoint; + v.label = m.label; + v.protectedRegion = m.protectedRegion; + v.__params = m.params || []; + return v; + }, + nearestKeyIndex: function (t) { + var l = this.__list(), best = 0, d = Infinity; + for (var i = 0; i < l.length; i++) if (Math.abs(l[i].time - t) < d) { d = Math.abs(l[i].time - t); best = i + 1; } + if (!best) throw __err("there are no markers"); + return best; + }, + setValueAtTime: function (t, mv) { + t = __num(t, "time"); + if (!(mv instanceof MarkerValue)) throw __err("value must be a MarkerValue"); + var p = this.__p({ time: t, duration: mv.duration || 0, comment: mv.comment, chapter: mv.chapter, url: mv.url, frameTarget: mv.frameTarget, protected: !!mv.protectedRegion, label: mv.label || 0 }); + p.cuePoint = mv.cuePointName ? { name: mv.cuePointName, navigation: !mv.eventCuePoint, params: mv.__params } : null; + var l = this.__list(), at = -1; + for (var i = 0; i < l.length; i++) if (Math.abs(l[i].time - t) < 1e-6) at = i; + if (at >= 0) p.index = at; + else p["new"] = true; + __call("markers.set", p); + }, + setValueAtKey: function (i, mv) { this.setValueAtTime(this.keyTime(i), mv); }, + removeKey: function (i) { this.__at(i); __call("markers.delete", this.__p({ index: i - 1 })); }, + toString: function () { return "[object Property]"; }, +}; + +// ------------------------------------------------------------------ layers + +function __layer(compId, id) { + if (id === null || id === undefined) return null; + var i = __get("layer", { comp: compId, layer: id }); + switch (i.kind) { + case "text": return new TextLayer(compId, id); + case "shape": return new ShapeLayer(compId, id); + case "camera": return new CameraLayer(compId, id); + case "light": return new LightLayer(compId, id); + default: return new AVLayer(compId, id); + } +} + +function Layer() {} +Layer.prototype = { + __init: function (comp, id, kind) { + this.__comp = comp; + this.__id = id; + this.__kind = kind; + return new Proxy(this, __attr); + }, + __info: function () { return __get("layer", { comp: this.__comp, layer: this.__id }); }, + __p: function (o) { + o = o || {}; + o.comp = this.__comp; + o.layers = [this.__id]; + return o; + }, + __p1: function (o) { + o = o || {}; + o.comp = this.__comp; + o.layer = this.__id; + return o; + }, + __switch: function (sw, v) { __call("layer.setSwitch", this.__p({ "switch": sw, value: !!v })); }, + // Comp seconds → layer seconds (keyframe times are layer time). + __layerTime: function (t) { + var i = this.__info(); + return (t - i.startTime) * 100 / (i.stretch || 100); + }, + toString: function () { return "[object " + this.__kind + "]"; }, + get id() { return this.__id; }, + get name() { return this.__info().name; }, + set name(v) { __call("layer.rename", this.__p1({ name: String(v) })); }, + get index() { return this.__info().index; }, + get matchName() { return this.__info().matchName; }, + get containingComp() { return new CompItem(this.__comp); }, + get propertyDepth() { return 0; }, + get propertyType() { return PropertyType.INDEXED_GROUP; }, + get parentProperty() { return null; }, + get isModified() { return false; }, + get elided() { return false; }, + get isEffect() { return false; }, + get isMask() { return false; }, + get canSetEnabled() { return true; }, + get enabled() { return this.__info().enabled; }, + set enabled(v) { this.__switch("video", v); }, + get active() { return this.__info().active; }, + get solo() { return this.__info().solo; }, + set solo(v) { this.__switch("solo", v); }, + get locked() { return this.__info().locked; }, + set locked(v) { this.__switch("lock", v); }, + get shy() { return this.__info().shy; }, + set shy(v) { this.__switch("shy", v); }, + get audioEnabled() { return this.__info().audioEnabled; }, + set audioEnabled(v) { this.__switch("audio", v); }, + get hasVideo() { return this.__info().hasVideo; }, + get hasAudio() { return this.__info().hasAudio; }, + get audioActive() { var i = this.__info(); return i.hasAudio && i.audioEnabled; }, + get nullLayer() { return this.__info().nullLayer; }, + get isNameSet() { return true; }, + get inPoint() { return this.__info().inPoint; }, + set inPoint(v) { __call("layer.timing", this.__p({ "in": __num(v, "inPoint") })); }, + get outPoint() { return this.__info().outPoint; }, + set outPoint(v) { __call("layer.timing", this.__p({ out: __num(v, "outPoint") })); }, + get startTime() { return this.__info().startTime; }, + set startTime(v) { __call("layer.timing", this.__p({ start: __num(v, "startTime") })); }, + get stretch() { return this.__info().stretch; }, + set stretch(v) { __call("layer.timeStretch", this.__p({ percent: __num(v, "stretch") })); }, + get time() { return this.__info().time; }, + get label() { return this.__info().label; }, + set label(v) { + var names = __get("labels"); + var n = names[__num(v, "label")]; + if (n === undefined) throw __err("label must be 0-16"); + __call("edit.label", this.__p({ label: n })); + }, + get comment() { return this.__info().comment; }, + get selected() { return this.__info().selected; }, + set selected(v) { + var sel = this.containingComp.__info().selectedLayers; + var on = sel.indexOf(this.__id) >= 0; + if (!!v !== on) __call("layer.select", { comp: this.__comp, layers: [this.__id], toggle: true }); + }, + get selectedProperties() { + var c = this.containingComp.__info().selectedProperties, out = []; + for (var i = 0; i < c.length; i++) if (c[i][0] === this.__id) out.push(__propOf(this, __get("node", { comp: this.__comp, layer: this.__id, uid: c[i][1] }))); + return out; + }, + get parent() { var p = this.__info().parent; return p === null ? null : __layer(this.__comp, p); }, + set parent(v) { __call("layer.setParent", this.__p({ parent: v ? v.__id : null })); }, + setParentWithJump: function (v) { __call("layer.setParent", this.__p({ parent: v ? v.__id : null, compensate: false })); }, + get autoOrient() { return AutoOrientType[{ Off: "NO_AUTO_ORIENT", AlongPath: "ALONG_PATH", TowardsCamera: "CHARACTERS_TOWARD_CAMERA", TowardsPointOfInterest: "CAMERA_OR_POINT_OF_INTEREST" }[this.__info().autoOrient]]; }, + set autoOrient(v) { + var m = { 4212: "off", 4213: "alongPath", 4214: "towardsPointOfInterest", 4215: "towardsCamera" }[v]; + if (!m) throw __err("Bad AutoOrientType value " + v); + __call("layer.autoOrient", this.__p({ mode: m })); + }, + get marker() { return new MarkerProperty(this.__comp, this); }, + get numProperties() { return __get("node", { comp: this.__comp, layer: this.__id, uid: 0 }).numProperties; }, + property: function (key) { + if (key === undefined || key === null) throw __err("property(): missing name or index"); + if (key === "Marker" || key === "ADBE Marker" || key === "marker") return this.marker; + var k = typeof key === "number" ? key : String(key); + return __propOf(this, __get("child", { comp: this.__comp, layer: this.__id, uid: 0, key: k })); + }, + propertyGroup: function () { return null; }, + canAddProperty: function () { return false; }, + addProperty: function (n) { throw __err("Can not add \"" + n + "\" to a layer: use a property group (Effects, Masks, Contents)"); }, + remove: function () { __call("edit.clear", this.__p()); }, + duplicate: function () { + var before = this.containingComp.__info().layers; + __call("edit.duplicate", this.__p()); + var after = this.containingComp.__info().layers; + for (var i = 0; i < after.length; i++) if (before.indexOf(after[i]) < 0) return __layer(this.__comp, after[i]); + return null; + }, + copyToComp: function (comp) { + var c = __call("edit.copy", this.__p()); + var before = comp.__info().layers; + __call("comp.open", { comp: comp.__id }); + __call("edit.paste", { comp: comp.__id }); + return c; + }, + moveToBeginning: function () { __call("layer.arrange", this.__p({ to: "front" })); }, + moveToEnd: function () { __call("layer.arrange", this.__p({ to: "back" })); }, + moveBefore: function (other) { + var me = this.index, to = other.index; + __call("layer.arrange", this.__p({ to: "index", index: me < to ? to - 1 : to })); + }, + moveAfter: function (other) { + var me = this.index, to = other.index; + __call("layer.arrange", this.__p({ to: "index", index: me < to ? to : to + 1 })); + }, + activeAtTime: function (t) { var i = this.__info(); return i.enabled && t >= i.inPoint && t < i.outPoint; }, + applyPreset: function (f) { __call("anim.applyPreset", this.__p({ path: __str(f) })); }, +}; + +function AVLayer(comp, id) { return this.__init(comp, id, "AVLayer"); } +AVLayer.prototype = Object.create(Layer.prototype); +AVLayer.prototype.constructor = AVLayer; +(function () { + var P = AVLayer.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + def("source", function () { return __item(this.__info().source); }); + def("width", function () { return this.__info().width; }); + def("height", function () { return this.__info().height; }); + def("threeDLayer", function () { return this.__info().threeDLayer; }, function (v) { this.__switch("threeD", v); }); + def("adjustmentLayer", function () { return this.__info().adjustmentLayer; }, function (v) { this.__switch("adjustment", v); }); + def("guideLayer", function () { return this.__info().guideLayer; }, function (v) { this.__switch("guide", v); }); + def("motionBlur", function () { return this.__info().motionBlur; }, function (v) { this.__switch("motionBlur", v); }); + def("effectsActive", function () { return this.__info().effectsActive; }, function (v) { this.__switch("fx", v); }); + def("collapseTransformation", function () { return this.__info().collapse; }, function (v) { this.__switch("collapse", v); }); + def("preserveTransparency", function () { return this.__info().preserveTransparency; }, function (v) { this.__switch("preserveTransparency", v); }); + def("quality", function () { return LayerQuality[this.__info().quality.toUpperCase()]; }, function (v) { + var q = __enumKey(LayerQuality, v); + if (!q) throw __err("Bad LayerQuality value " + v); + __call("layer.quality", this.__p({ quality: q.toLowerCase() })); + }); + def("samplingQuality", function () { return LayerSamplingQuality[this.__info().samplingQuality.toUpperCase()]; }, function (v) { + var q = __enumKey(LayerSamplingQuality, v); + if (!q) throw __err("Bad LayerSamplingQuality value " + v); + __call("layer.sampling", this.__p({ sampling: q.toLowerCase() })); + }); + def("frameBlendingType", function () { return { Off: 4012, FrameMix: 4013, PixelMotion: 4014 }[this.__info().frameBlending]; }, function (v) { + var m = { 4012: "off", 4013: "frameMix", 4014: "pixelMotion" }[v]; + if (!m) throw __err("Bad FrameBlendingType value " + v); + __call("layer.frameBlending", this.__p({ mode: m })); + }); + def("frameBlending", function () { return this.__info().frameBlending !== "Off"; }); + def("blendingMode", function () { return __blendNames[this.__info().blendingMode]; }, function (v) { + __call("layer.setBlendMode", this.__p({ mode: __blendLabel(v) })); + }); + def("trackMatteType", function () { + var m = this.__info().trackMatte; + return m ? __matteEnum[m.kind] : TrackMatteType.NO_TRACK_MATTE; + }, function (v) { + if (v === TrackMatteType.NO_TRACK_MATTE) { __call("layer.trackMatte", this.__p({ op: "none" })); return; } + var k = __matteKind[v]; + if (!k) throw __err("Bad TrackMatteType value " + v); + var m = this.__info().trackMatte; + if (m) __call("layer.setTrackMatte", this.__p1({ matte: m.layer, kind: k })); + else __call("layer.trackMatte", this.__p({ op: k })); + }); + def("trackMatteLayer", function () { var m = this.__info().trackMatte; return m ? __layer(this.__comp, m.layer) : null; }); + def("hasTrackMatte", function () { return !!this.__info().trackMatte; }); + def("isTrackMatte", function () { return this.__info().isTrackMatte; }); + def("timeRemapEnabled", function () { return this.__info().timeRemapEnabled; }, function (v) { + __call("layer.enableTimeRemap", this.__p({ value: !!v })); + }); + def("canSetTimeRemapEnabled", function () { var s = this.source; return !!s && !(s instanceof FootageItem && s.mainSource.isStill); }); + def("canSetCollapseTransformation", function () { return true; }); + def("isNameFromSource", function () { var s = this.source; return !!s && s.name === this.name; }); + def("environmentLayer", function () { return false; }); + P.setTrackMatte = function (layer, type) { + if (!layer || type === TrackMatteType.NO_TRACK_MATTE) { this.removeTrackMatte(); return; } + var k = __matteKind[type]; + if (!k) throw __err("Bad TrackMatteType value " + type); + __call("layer.setTrackMatte", this.__p1({ matte: layer.__id, kind: k })); + }; + P.removeTrackMatte = function () { __call("layer.setTrackMatte", this.__p1({ matte: null })); }; + P.sourceRectAtTime = function (t, extents) { + return __get("sourceRect", { comp: this.__comp, layer: this.__id, time: __num(t, "time"), extents: !!extents }); + }; + P.sourcePointToComp = function (p) { return p; }; + P.compPointToSource = function (p) { return p; }; + P.replaceSource = function () { throw __err("replaceSource is not supported yet"); }; + P.openInViewer = function () { __call("layer.openLayer", this.__p1()); return null; }; +})(); + +function TextLayer(comp, id) { return this.__init(comp, id, "TextLayer"); } +TextLayer.prototype = Object.create(AVLayer.prototype); +TextLayer.prototype.constructor = TextLayer; +Object.defineProperty(TextLayer.prototype, "source", { get: function () { return null; } }); + +function ShapeLayer(comp, id) { return this.__init(comp, id, "ShapeLayer"); } +ShapeLayer.prototype = Object.create(AVLayer.prototype); +ShapeLayer.prototype.constructor = ShapeLayer; +Object.defineProperty(ShapeLayer.prototype, "source", { get: function () { return null; } }); + +function CameraLayer(comp, id) { return this.__init(comp, id, "CameraLayer"); } +CameraLayer.prototype = Object.create(Layer.prototype); +CameraLayer.prototype.constructor = CameraLayer; + +function LightLayer(comp, id) { return this.__init(comp, id, "LightLayer"); } +LightLayer.prototype = Object.create(Layer.prototype); +LightLayer.prototype.constructor = LightLayer; +Object.defineProperty(LightLayer.prototype, "lightType", { + get: function () { return LightType[this.__info().lightType.toUpperCase()]; }, + set: function (v) { + var k = __enumKey(LightType, v); + if (!k) throw __err("Bad LightType value " + v); + __call("layer.lightSettings", this.__p1({ kind: k.charAt(0) + k.substring(1).toLowerCase() })); + }, +}); + +// ------------------------------------------------------------------ collections + +function LayerCollection(comp) { + this.__comp = comp; + return new Proxy(this, __coll); +} +LayerCollection.prototype = { + constructor: LayerCollection, + __ids: function () { return __get("item", { id: this.__comp }).layers; }, + get length() { return this.__ids().length; }, + __at: function (i) { var ids = this.__ids(); return i >= 1 && i <= ids.length ? __layer(this.__comp, ids[i - 1]) : undefined; }, + __new: function (r) { return __layer(this.__comp, r.layer); }, + __dur: function (layer, duration) { + if (duration !== undefined && duration !== null) { + var i = layer.__info(); + __call("layer.timing", { comp: this.__comp, layers: [layer.__id], out: i.inPoint + __num(duration, "duration") }); + } + return layer; + }, + add: function (item, duration) { + if (!item || item.id === undefined) throw __err("add(): expected an item"); + var l = this.__new(__call("layer.addItem", { comp: this.__comp, item: item.id, time: 0 })); + return this.__dur(l, duration); + }, + addSolid: function (color, name, width, height, pixelAspect, duration) { + var p = { comp: this.__comp, color: [color[0], color[1], color[2]] }; + if (name !== undefined) p.name = String(name); + if (width !== undefined) p.width = Math.round(__num(width, "width")); + if (height !== undefined) p.height = Math.round(__num(height, "height")); + return this.__dur(this.__new(__call("layer.newSolid", p)), duration); + }, + addNull: function (duration) { return this.__dur(this.__new(__call("layer.newNull", { comp: this.__comp })), duration); }, + addText: function (text) { + var t = text instanceof TextDocument ? text : new TextDocument(text === undefined ? "" : String(text)); + var p = t.__attrs(); + p.comp = this.__comp; + if (p.text === undefined) p.text = t.text; + return this.__new(__call("layer.newText", p)); + }, + addBoxText: function (size, text) { + var c = __get("item", { id: this.__comp }); + var w = size[0], h = size[1]; + var t = text instanceof TextDocument ? text : new TextDocument(text === undefined ? "" : String(text)); + var p = t.__attrs(); + p.comp = this.__comp; + if (p.text === undefined) p.text = t.text; + p.box = [(c.width - w) / 2, (c.height - h) / 2, w, h]; + return this.__new(__call("layer.newText", p)); + }, + addShape: function () { return this.__new(__call("layer.newShape", { comp: this.__comp, kind: "none" })); }, + addCamera: function (name, center) { + var p = { comp: this.__comp, name: String(name) }; + if (center) p.poi = [center[0], center[1], 0]; + return this.__new(__call("layer.newCamera", p)); + }, + addLight: function (name, center) { + var p = { comp: this.__comp, name: String(name), kind: "Point" }; + if (center) p.poi = [center[0], center[1], 0]; + return this.__new(__call("layer.newLight", p)); + }, + precompose: function (indices, name, moveAllAttributes) { + var ids = this.__ids(), sel = []; + for (var i = 0; i < indices.length; i++) sel.push(ids[indices[i] - 1]); + var r = __call("layer.precompose", { comp: this.__comp, layers: sel, name: String(name), mode: moveAllAttributes === false ? "leave" : "move" }); + return new CompItem(r.comp); + }, + byName: function (name) { + var ids = this.__ids(); + for (var i = 0; i < ids.length; i++) { + var l = __get("layer", { comp: this.__comp, layer: ids[i] }); + if (l.name === name) return __layer(this.__comp, ids[i]); + } + return null; + }, + toString: function () { return "[object LayerCollection]"; }, +}; + +function ItemCollection(folder) { + this.__folder = folder; + return new Proxy(this, __coll); +} +ItemCollection.prototype = { + constructor: ItemCollection, + __ids: function () { + if (this.__folder === 0) return __get("project").items; + return __get("item", { id: this.__folder }).children; + }, + get length() { return this.__ids().length; }, + __at: function (i) { var ids = this.__ids(); return i >= 1 && i <= ids.length ? __item(ids[i - 1]) : undefined; }, + __place: function (id) { + if (this.__folder) __call("project.move", { items: [id], folder: this.__folder }); + return __item(id); + }, + addComp: function (name, width, height, pixelAspect, duration, frameRate) { + var r = __call("comp.new", { + name: String(name), width: Math.round(__num(width, "width")), height: Math.round(__num(height, "height")), + pixelAspect: __num(pixelAspect, "pixelAspect"), duration: __num(duration, "duration"), frameRate: __num(frameRate, "frameRate"), open: false, + }); + return this.__place(r.comp); + }, + addFolder: function (name) { + var r = __call("project.newFolder", { name: String(name) }); + var id = r && typeof r === "object" ? (r.folder || r.item || r.id) : r; + return this.__place(id); + }, + toString: function () { return "[object ItemCollection]"; }, +}; + +// ------------------------------------------------------------------ items + +function __item(id) { + if (id === null || id === undefined) return null; + if (id === 0) return new FolderItem(0); + var i = __get("item", { id: id }); + switch (i.kind) { + case "comp": return new CompItem(id); + case "folder": return new FolderItem(id); + default: return new FootageItem(id); + } +} + +function Item() {} +Item.prototype = { + __info: function () { return __get("item", { id: this.__id }); }, + toString: function () { return "[object " + this.__kind + "]"; }, + get id() { return this.__id; }, + get name() { return this.__info().name; }, + set name(v) { __call("project.rename", { item: this.__id, name: String(v) }); }, + get typeName() { return this.__info().typeName; }, + get comment() { return this.__info().comment; }, + set comment(v) { __call("project.setComment", { items: [this.__id], comment: String(v) }); }, + get label() { return this.__info().label; }, + set label(v) { __call("project.setLabel", { items: [this.__id], label: __num(v, "label") }); }, + get parentFolder() { return __item(this.__info().parent); }, + set parentFolder(f) { __call("project.move", { items: [this.__id], folder: f && f.__id ? f.__id : null }); }, + get selected() { return this.__info().selected; }, + set selected(v) { if (v) __call("project.select", { items: [this.__id], add: true }); }, + get usedIn() { return this.__info().usedIn.map(function (c) { return new CompItem(c); }); }, + remove: function () { __call("project.delete", { items: [this.__id] }); }, +}; + +function AVItem() {} +AVItem.prototype = Object.create(Item.prototype); +(function () { + function def(name, get, set) { Object.defineProperty(AVItem.prototype, name, { get: get, set: set }); } + def("width", function () { return this.__info().width; }, function (v) { this.__settings({ width: Math.round(__num(v, "width")) }); }); + def("height", function () { return this.__info().height; }, function (v) { this.__settings({ height: Math.round(__num(v, "height")) }); }); + def("pixelAspect", function () { return this.__info().pixelAspect; }, function (v) { this.__settings({ pixelAspect: __num(v) }); }); + def("duration", function () { return this.__info().duration; }, function (v) { this.__settings({ duration: __num(v, "duration") }); }); + def("frameRate", function () { return this.__info().frameRate; }, function (v) { this.__settings({ frameRate: __num(v, "frameRate") }); }); + def("frameDuration", function () { var f = this.__info().frameRate; return f ? 1 / f : 0; }, function (v) { this.__settings({ frameRate: 1 / __num(v) }); }); + def("hasVideo", function () { return this.__info().hasVideo; }); + def("hasAudio", function () { return this.__info().hasAudio; }); + def("footageMissing", function () { return !!this.__info().footageMissing; }); + def("useProxy", function () { return false; }); + def("proxySource", function () { return null; }); + def("isMediaReplacementCompatible", function () { return false; }); + def("time", function () { return 0; }); +})(); +AVItem.prototype.__settings = function () { throw __err("can't change this item's settings from a script"); }; + +function CompItem(id) { + this.__id = id; + this.__kind = "CompItem"; +} +CompItem.prototype = Object.create(AVItem.prototype); +CompItem.prototype.constructor = CompItem; +(function () { + var P = CompItem.prototype; + function def(name, get, set) { Object.defineProperty(P, name, { get: get, set: set }); } + P.__settings = function (o) { o.comp = this.__id; __call("comp.settings", o); }; + def("typeName", function () { return "Composition"; }); + def("layers", function () { return new LayerCollection(this.__id); }); + def("numLayers", function () { return this.__info().layers.length; }); + def("selectedLayers", function () { var c = this.__id; return this.__info().selectedLayers.map(function (l) { return __layer(c, l); }); }); + def("selectedProperties", function () { + var c = this.__id, out = []; + this.__info().selectedProperties.forEach(function (lp) { + var L = __layer(c, lp[0]); + out.push(__propOf(L, __get("node", { comp: c, layer: lp[0], uid: lp[1] }))); + }); + return out; + }); + def("activeCamera", function () { + var ids = this.__info().layers; + for (var i = 0; i < ids.length; i++) { + var l = __get("layer", { comp: this.__id, layer: ids[i] }); + if (l.kind === "camera" && l.enabled && l.active) return __layer(this.__id, ids[i]); + } + return null; + }); + def("time", function () { return this.__info().time; }, function (v) { __call("time.set", { comp: this.__id, time: __num(v, "time") }); }); + def("workAreaStart", function () { return this.__info().workAreaStart; }, function (v) { + var i = this.__info(); + __call("comp.workArea", { comp: this.__id, start: __num(v), end: __num(v) + i.workAreaDuration }); + }); + def("workAreaDuration", function () { return this.__info().workAreaDuration; }, function (v) { + var i = this.__info(); + __call("comp.workArea", { comp: this.__id, start: i.workAreaStart, end: i.workAreaStart + __num(v) }); + }); + def("bgColor", function () { return this.__info().bgColor; }, function (v) { this.__settings({ background: [v[0], v[1], v[2]] }); }); + def("displayStartTime", function () { return this.__info().displayStartTime; }, function (v) { this.__settings({ startTime: __num(v) }); }); + def("hideShyLayers", function () { return this.__info().hideShyLayers; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "hideShy", value: !!v }); }); + def("motionBlur", function () { return this.__info().motionBlur; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "motionBlur", value: !!v }); }); + def("frameBlending", function () { return this.__info().frameBlending; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "frameBlending", value: !!v }); }); + def("draft3d", function () { return this.__info().draft3d; }, function (v) { __call("comp.setSwitch", { comp: this.__id, "switch": "draft3d", value: !!v }); }); + def("shutterAngle", function () { return this.__info().shutterAngle; }, function (v) { this.__settings({ shutterAngle: __num(v) }); }); + def("shutterPhase", function () { return this.__info().shutterPhase; }, function (v) { this.__settings({ shutterPhase: __num(v) }); }); + def("motionBlurSamplesPerFrame", function () { return this.__info().motionBlurSamplesPerFrame; }, function (v) { this.__settings({ motionBlurSamples: __num(v) }); }); + def("renderer", function () { return this.__info().renderer; }, function (v) { + this.__settings({ renderer: String(v).toLowerCase().indexOf("advanced") >= 0 ? "advanced3D" : "classic3D" }); + }); + def("renderers", function () { return ["ADBE Classic 3D", "ADBE Advanced 3d"]; }); + def("markerProperty", function () { return new MarkerProperty(this.__id, null); }); + def("hasVideo", function () { return true; }); + P.layer = function (a, b) { + var ids = this.__info().layers; + if (typeof a === "number") { + if (b !== undefined) { + // layer(otherLayer, relIndex) + throw __err("layer(index, relIndex) is not supported"); + } + if (a < 1 || a > ids.length || Math.floor(a) !== a) throw __err("layer index " + a + " is out of range 1.." + ids.length); + return __layer(this.__id, ids[a - 1]); + } + if (a && a.__id !== undefined && typeof b === "number") { + var j = a.index + b; + return j >= 1 && j <= ids.length ? __layer(this.__id, ids[j - 1]) : null; + } + return new LayerCollection(this.__id).byName(String(a)); + }; + P.openInViewer = function () { __call("comp.open", { comp: this.__id }); return null; }; + P.duplicate = function () { return __item(__call("project.duplicate", { items: [this.__id] }).items[0]); }; + P.exportAsMotionGraphicsTemplate = function () { return false; }; +})(); + +function FootageItem(id) { + this.__id = id; + this.__kind = "FootageItem"; +} +FootageItem.prototype = Object.create(AVItem.prototype); +FootageItem.prototype.constructor = FootageItem; +Object.defineProperty(FootageItem.prototype, "file", { + get: function () { var f = this.__info().file; return f ? new File(f) : null; }, +}); +Object.defineProperty(FootageItem.prototype, "mainSource", { + get: function () { + var i = this.__info(); + if (i.kind === "solid") return { isStill: true, color: i.color, hasAlpha: false, toString: function () { return "[object SolidSource]"; } }; + return { isStill: !!i.isStill, file: i.file ? new File(i.file) : null, missingFootagePath: i.footageMissing ? i.file : "", loop: i.loop, hasAlpha: false, toString: function () { return "[object FileSource]"; } }; + }, +}); +FootageItem.prototype.replace = function (f) { __call("file.replaceFootage", { item: this.__id, path: __str(f) }); }; +FootageItem.prototype.replaceWithSolid = function (color, name, w, h) { + __call("file.replaceWithSolid", { item: this.__id, color: [color[0], color[1], color[2]], width: w, height: h }); + if (name !== undefined) this.name = name; +}; +FootageItem.prototype.openInViewer = function () { return null; }; + +function FolderItem(id) { + this.__id = id; + this.__kind = "FolderItem"; +} +FolderItem.prototype = Object.create(Item.prototype); +FolderItem.prototype.constructor = FolderItem; +(function () { + var P = FolderItem.prototype; + P.__info = function () { + if (this.__id === 0) return { id: 0, name: "Root", typeName: "Folder", comment: "", label: 0, parent: null, selected: false, usedIn: [], children: [] }; + return __get("item", { id: this.__id }); + }; + Object.defineProperty(P, "items", { get: function () { return new ItemCollection(this.__id); } }); + Object.defineProperty(P, "numItems", { get: function () { return new ItemCollection(this.__id).length; } }); + Object.defineProperty(P, "parentFolder", { + get: function () { return this.__id === 0 ? null : __item(this.__info().parent); }, + set: function (f) { __call("project.move", { items: [this.__id], folder: f && f.__id ? f.__id : null }); }, + }); + P.item = function (i) { + var v = new ItemCollection(this.__id).__at(__num(i, "index")); + if (v === undefined) throw __err("item index " + i + " is out of range"); + return v; + }; +})(); + +// ------------------------------------------------------------------ render queue + +function OutputModule(rq, n) { + this.__rq = rq; + this.__n = n; +} +OutputModule.prototype = { + constructor: OutputModule, + __info: function () { return this.__rq.__info(); }, + get name() { var i = this.__info(); return this.__n === 1 ? i.outputModuleSummary : i.extraOutputs[this.__n - 2].summary; }, + get file() { + var i = this.__info(); + var p = this.__n === 1 ? i.outputPath : i.extraOutputs[this.__n - 2].outputPath; + return p ? new File(p) : null; + }, + set file(f) { __call("renderQueue.setOutput", { item: this.__rq.__id, module: this.__n, path: __str(f) }); }, + get templates() { return ["H.264", "ProRes", "PNG Sequence", "JPEG Sequence", "TIFF Sequence", "OpenEXR Sequence", "GIF"]; }, + applyTemplate: function (name) { + var n = String(name).toLowerCase(); + var f = n.indexOf("prores") >= 0 ? "prores" : n.indexOf("png") >= 0 || n.indexOf("lossless") >= 0 ? "png" : n.indexOf("jpeg") >= 0 || n.indexOf("jpg") >= 0 ? "jpeg" + : n.indexOf("tiff") >= 0 ? "tiff" : n.indexOf("exr") >= 0 ? "exr" : n.indexOf("gif") >= 0 ? "gif" : n.indexOf("264") >= 0 ? "h264" : null; + if (!f) throw __err("Unknown output module template \"" + name + "\""); + __call("renderQueue.setOutputModule", { item: this.__rq.__id, module: this.__n, format: f }); + }, + remove: function () { throw __err("removing output modules is not supported"); }, + toString: function () { return "[object OutputModule]"; }, +}; + +var __rqStatus = { "Unqueued": 3014, "Queued": 3015, "Needs Output": 3013, "Rendering": 3016, "Done": 3019, "User Stopped": 3017, "Failed": 3018 }; + +function RenderQueueItem(id) { this.__id = id; } +RenderQueueItem.prototype = { + constructor: RenderQueueItem, + __info: function () { + var l = __call("renderQueue.list", {}).items; + for (var i = 0; i < l.length; i++) if (l[i].id === this.__id) return l[i]; + throw __err("the render queue item no longer exists"); + }, + get comp() { return new CompItem(this.__info().comp); }, + get status() { + var s = this.__info().statusLabel; + for (var k in __rqStatus) if (s.indexOf(k) === 0) return __rqStatus[k]; + return RQItemStatus.QUEUED; + }, + get render() { return this.__info().render; }, + set render(v) { __call("renderQueue.setRender", { item: this.__id, render: !!v }); }, + get numOutputModules() { return this.__info().outputModules; }, + get outputModules() { + var n = this.numOutputModules, out = []; + for (var i = 1; i <= n; i++) out.push(new OutputModule(this, i)); + return out; + }, + outputModule: function (i) { + if (i < 1 || i > this.numOutputModules) throw __err("output module index " + i + " is out of range"); + return new OutputModule(this, i); + }, + get timeSpanStart() { var s = this.__info().settings; return s.start !== undefined ? s.start : 0; }, + set timeSpanStart(v) { __call("renderQueue.setRenderSettings", { item: this.__id, timeSpan: "custom", start: __num(v), end: __num(v) + this.timeSpanDuration }); }, + get timeSpanDuration() { var s = this.__info().settings; return s.end !== undefined && s.start !== undefined ? s.end - s.start : this.comp.duration; }, + set timeSpanDuration(v) { var s = this.timeSpanStart; __call("renderQueue.setRenderSettings", { item: this.__id, timeSpan: "custom", start: s, end: s + __num(v) }); }, + get elapsedSeconds() { return this.__info().render_time || 0; }, + get startTime() { var s = this.__info().started; return s ? new Date(s * 1000) : null; }, + get logType() { return LogType.ERRORS_ONLY; }, + set logType(v) {}, + get skipFrames() { return 0; }, + outputModuleAdd: function () { __call("render.addOutputModule", { item: this.__id }); }, + remove: function () { __call("renderQueue.remove", { item: this.__id }); }, + duplicate: function () { + var before = __call("renderQueue.list", {}).items.map(function (i) { return i.id; }); + __call("renderQueue.duplicate", { item: this.__id }); + var after = __call("renderQueue.list", {}).items; + for (var i = 0; i < after.length; i++) if (before.indexOf(after[i].id) < 0) return new RenderQueueItem(after[i].id); + return null; + }, + applyTemplate: function () {}, + getSettings: function () { return this.__info().settings; }, + setSettings: function (o) { __call("renderQueue.setRenderSettings", Object.assign({ item: this.__id }, o)); }, + toString: function () { return "[object RenderQueueItem]"; }, +}; + +function RQItemCollection() { return new Proxy(this, __coll); } +RQItemCollection.prototype = { + constructor: RQItemCollection, + __list: function () { return __call("renderQueue.list", {}).items; }, + get length() { return this.__list().length; }, + __at: function (i) { var l = this.__list(); return i >= 1 && i <= l.length ? new RenderQueueItem(l[i - 1].id) : undefined; }, + add: function (comp) { + if (!(comp instanceof CompItem)) throw __err("add(): expected a CompItem"); + __call("renderQueue.add", { comp: comp.__id }); + var l = this.__list(); + return new RenderQueueItem(l[l.length - 1].id); + }, + toString: function () { return "[object RQItemCollection]"; }, +}; + +function RenderQueue() {} +RenderQueue.prototype = { + constructor: RenderQueue, + get items() { return new RQItemCollection(); }, + get numItems() { return new RQItemCollection().length; }, + get rendering() { return !!__call("renderQueue.list", {}).rendering; }, + get canQueueInAME() { return false; }, + get queueNotify() { return false; }, + set queueNotify(v) {}, + item: function (i) { + var v = new RQItemCollection().__at(__num(i, "index")); + if (v === undefined) throw __err("render queue index " + i + " is out of range"); + return v; + }, + render: function () { __call("renderQueue.render", { wait: true }); }, + stopRendering: function () { __call("renderQueue.stop", {}); }, + pauseRendering: function () {}, + showWindow: function () {}, + queueInAME: function () { throw __err("Adobe Media Encoder is not available: render with app.project.renderQueue.render()"); }, + toString: function () { return "[object RenderQueue]"; }, +}; + +// ------------------------------------------------------------------ project and app + +function ImportOptions(file) { + if (!(this instanceof ImportOptions)) return new ImportOptions(file); + this.file = file || null; + this.importAs = ImportAsType.FOOTAGE; + this.sequence = false; + this.forceAlphabetical = false; + this.rangeStart = 0; + this.rangeEnd = 0; +} +ImportOptions.prototype.canImportAs = function (t) { return t === ImportAsType.FOOTAGE; }; + +function Project() {} +Project.prototype = { + constructor: Project, + __info: function () { return __get("project"); }, + get items() { return new ItemCollection(0); }, + get numItems() { return this.__info().items.length; }, + get rootFolder() { return new FolderItem(0); }, + get activeItem() { return __item(this.__info().active); }, + get selection() { return this.__info().selection.map(__item); }, + get file() { var p = this.__info().path; return p ? new File(p) : null; }, + get dirty() { return this.__info().dirty; }, + get renderQueue() { return new RenderQueue(); }, + get bitsPerChannel() { return this.__info().bitsPerChannel; }, + set bitsPerChannel(v) { __call("file.projectSettings", { bitDepth: __num(v) }); }, + get linearBlending() { return this.__info().linearBlending; }, + set linearBlending(v) { __call("file.projectSettings", { blendLinear: !!v }); }, + get linearizeWorkingSpace() { return this.__info().linearizeWorkingSpace; }, + set linearizeWorkingSpace(v) { __call("file.projectSettings", { linearize: !!v }); }, + get workingSpace() { return this.__info().workingSpace; }, + set workingSpace(v) { __call("file.projectSettings", { workingSpace: String(v) || "none" }); }, + get timeDisplayType() { return this.__info().timeDisplayFrames ? TimeDisplayType.FRAMES : TimeDisplayType.TIMECODE; }, + set timeDisplayType(v) { __call("file.projectSettings", { timeDisplay: v === TimeDisplayType.FRAMES ? "frames" : "timecode" }); }, + get expressionEngine() { return "javascript-1.0"; }, + set expressionEngine(v) {}, + item: function (i) { + var v = new ItemCollection(0).__at(__num(i, "index")); + if (v === undefined) throw __err("item index " + i + " is out of range 1.." + this.numItems); + return v; + }, + itemByID: function (id) { + try { return __item(__num(id)); } catch (e) { return null; } + }, + layerByID: function () { return null; }, + save: function (file) { + if (file) __call("file.saveAs", { path: __str(file) }); + else __call("file.save", {}); + return true; + }, + saveAs: function (file) { return this.save(file); }, + saveWithDialog: function () { if (this.file) return this.save(); return false; }, + close: function (opt) { + if (opt === CloseOptions.SAVE_CHANGES && this.file) this.save(); + __call("file.newProject", {}); + return true; + }, + importFile: function (opts) { + var f = opts instanceof ImportOptions ? opts.file : opts; + if (!f) throw __err("importFile(): no file"); + var r = __call("file.import", { paths: [__str(f)] }); + if (r.errors && r.errors.length) throw __err(r.errors.join("; ")); + return __item(r.items[0]); + }, + importFileWithDialog: function () { return null; }, + importPlaceholder: function (name, w, h, rate, dur) { + var r = __call("file.importPlaceholder", { name: String(name), width: w, height: h, frameRate: rate, duration: dur }); + var id = r && typeof r === "object" ? (r.item || r.id) : r; + return __item(id); + }, + consolidateFootage: function () { return __call("file.consolidateFootage", {}); }, + removeUnusedFootage: function () { return __call("file.removeUnusedFootage", {}); }, + reduceProject: function (items) { + __call("project.select", { items: items.map(function (i) { return i.__id; }) }); + return __call("file.reduceProject", {}); + }, + autoFixExpressions: function () {}, + showWindow: function () {}, + toString: function () { return "[object Project]"; }, +}; + +var __tasks = {}; +var app = { + get project() { return new Project(); }, + get version() { return __get("app").version; }, + get buildName() { return __get("app").buildName; }, + get buildNumber() { return 0; }, + get isWatchFolder() { return false; }, + get isRenderEngine() { return false; }, + get isoLanguage() { return "en_US"; }, + get language() { return 0; }, + get memoryInUse() { return 0; }, + get effects() { return __get("effects").map(function (e) { return { displayName: e.name, matchName: e.matchName, category: e.category, version: "1.0" }; }); }, + get activeViewer() { return null; }, + get availableGPUAccelTypes() { return []; }, + exitAfterLaunchAndEval: false, + exitCode: 0, + saveProjectOnCrash: true, + onError: null, + settings: { + __s: {}, + haveSetting: function (sec, key) { return (sec + "/" + key) in this.__s; }, + getSetting: function (sec, key) { return this.__s[sec + "/" + key] || ""; }, + saveSetting: function (sec, key, v) { this.__s[sec + "/" + key] = String(v); }, + }, + preferences: { + havePref: function () { return false; }, + getPrefAsString: function () { return ""; }, + getPrefAsLong: function () { return 0; }, + getPrefAsBool: function () { return false; }, + getPrefAsFloat: function () { return 0; }, + savePrefAsString: function () {}, + savePrefAsLong: function () {}, + savePrefAsBool: function () {}, + savePrefAsFloat: function () {}, + reload: function () {}, + }, + newProject: function () { __call("file.newProject", {}); return new Project(); }, + open: function (file) { + if (!file) return null; + __call("file.open", { path: __str(file) }); + return new Project(); + }, + quit: function () {}, + beginUndoGroup: function (name) { __undo("begin", name === undefined ? "Script" : String(name)); }, + endUndoGroup: function () { __undo("end", ""); }, + beginSuppressDialogs: function () {}, + endSuppressDialogs: function () {}, + purge: function () { __call("edit.purge", { what: "all" }); }, + findMenuCommandId: function (name) { return __get("menuCommandId", { name: String(name) }); }, + executeCommand: function (id) { + var cmd = typeof id === "number" ? __get("commandAt", { index: id }) : String(id); + if (!cmd) throw __err("Unknown command id " + id); + return __call(cmd, {}); + }, + scheduleTask: function (code, delay, repeat) { return __task("schedule", String(code), !!repeat); }, + cancelTask: function (id) { __task("cancel", id); }, + setMemoryUsageLimits: function () {}, + setMultiFrameRenderingConfig: function () {}, + setSavePreferencesOnQuit: function () {}, + activate: function () {}, + watchFolder: function () { throw __err("watch folders are not supported"); }, + pauseWatchFolder: function () {}, + endWatchFolder: function () {}, + parseSwatchFile: function () { throw __err("parseSwatchFile is not supported"); }, + // EffectCraft: run any engine command by id with parameters (see `command.list`). + run: function (id, params) { return __call(String(id), params || {}); }, + toString: function () { return "[object Application]"; }, +}; + +// The completion value of a script, as JSON text for the host. +function __result(v) { + if (v === undefined) return undefined; + if (v === null || typeof v === "number" || typeof v === "boolean" || typeof v === "string") return JSON.stringify(v); + if (v instanceof Layer || v instanceof Item || v instanceof PropertyBase || v instanceof Project || v instanceof TextDocument) { + var d = { type: v.toString() }; + try { d.name = v.name; } catch (e) {} + if (v instanceof TextDocument) d.text = v.text; + return JSON.stringify(d); + } + try { + var s = JSON.stringify(v); + return s === undefined ? JSON.stringify(String(v)) : s; + } catch (e) { + return JSON.stringify(String(v)); + } +} diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs new file mode 100644 index 000000000..61b495950 --- /dev/null +++ b/crates/script/src/runtime.rs @@ -0,0 +1,546 @@ +//! The boa runtime: native functions (`__exec`, `__query`, `__print`, `__undo`, `__task`, +//! `__file`), the session hand-off while a script runs, undo groups and error reporting. + +use std::cell::RefCell; +use std::mem::ManuallyDrop; +use std::sync::Arc; + +use boa_engine::{Context, JsNativeError, JsResult, JsString, JsValue, NativeFunction, Source, js_string}; +use effectcraft_engine::project::Project; +use effectcraft_engine::{ScriptRequest, Session}; +use serde_json::{Value as J, json}; + +const PRELUDE: &str = include_str!("prelude.js"); + +/// Engine commands that create layers: like After Effects, scripts add new layers at the top of +/// the stack (the UI inserts above the selection). +const CREATES_LAYER: &[&str] = + &["layer.newSolid", "layer.newNull", "layer.newText", "layer.newShape", "layer.newAdjustment", "layer.newCamera", "layer.newLight", "layer.addItem"]; + +/// What a script run produced. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Outcome { + /// The value of the script's last expression statement (JSON when it converts; model objects + /// as strings like `"[object CompItem]"`). + pub result: J, + /// `writeLn` / `$.writeln` / `alert` output, one entry per line. + pub output: Vec, + pub error: Option, +} + +/// An uncaught script error. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScriptError { + pub message: String, + pub line: Option, + pub column: Option, + pub file: String, +} + +impl Outcome { + pub fn ok(&self) -> bool { + self.error.is_none() + } + pub fn to_json(&self) -> J { + json!({ + "ok": self.ok(), + "result": self.result, + "output": self.output.join("\n"), + "error": self.error.as_ref().map(|e| json!({"message": e.message, "line": e.line, "column": e.column, "file": e.file})), + }) + } +} + +/// The session and per-run state while a script runs (the natives reach it here; nothing +/// borrowed ever enters the JS heap). +struct Active { + session: Session, + output: Vec, + /// Open undo groups: (name, project snapshot when the outermost group began). + groups: Vec<(String, Arc)>, + /// `app.scheduleTask` queue: (id, code, repeat). + tasks: Vec<(u32, String, bool)>, + next_task: u32, + name: String, +} + +thread_local! { + static ACTIVE: RefCell> = const { RefCell::new(None) }; + /// The Script Console's persistent context (never dropped, like the expression runtime: boa's + /// own thread-local heap may be torn down first at thread exit). + static CONSOLE: RefCell>> = const { RefCell::new(None) }; +} + +fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { + ACTIVE.with(|a| match a.try_borrow_mut() { + Ok(mut g) => match g.as_mut() { + Some(act) => f(act), + None => Err("no script is running".into()), + }, + Err(_) => Err("scripts can't run scripts".into()), + }) +} + +fn throw(msg: impl Into) -> boa_engine::JsError { + JsNativeError::error().with_message(msg.into()).into() +} + +fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { + Ok(args.get(i).cloned().unwrap_or_default().to_string(ctx)?.to_std_string_escaped()) +} + +fn arg_json(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { + let s = arg_string(args, i, ctx)?; + if s.is_empty() || s == "undefined" { + return Ok(json!({})); + } + serde_json::from_str(&s).map_err(|e| throw(format!("bad parameters: {e}"))) +} + +fn js_str(s: &str) -> JsValue { + JsValue::from(JsString::from(s)) +} + +/// `__exec(commandId, paramsJson)` → result JSON. Runs an engine command (undoable) with the +/// target comp made active for its duration. +fn native_exec(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let id = arg_string(args, 0, ctx)?; + let params = arg_json(args, 1, ctx)?; + if id == "script.run" || id == "file.runScript" { + return Err(throw("scripts can't run scripts")); + } + let r = with_active(|a| Ok(exec(&mut a.session, &id, params))); + match r { + Ok(Ok(v)) => Ok(js_str(&v.to_string())), + Ok(Err(e)) | Err(e) => Err(throw(e)), + } +} + +/// Run a command for a script: the `comp` it targets is active while it runs (commands act on +/// the active comp), then the user's active comp comes back. +pub(crate) fn exec(s: &mut Session, id: &str, params: J) -> Result { + use effectcraft_engine::project::ItemId; + let target = params.get("comp").and_then(J::as_u64).map(ItemId).filter(|c| s.project.comp(*c).is_some()); + let prev = s.state.active_comp; + if let Some(c) = target + && prev != Some(c) + { + s.state.active_comp = Some(c); + } + if CREATES_LAYER.contains(&id) { + s.state.selected_layers.clear(); + } + let r = s.execute(id, params); + if let Some(c) = target + && prev != Some(c) + && s.state.active_comp == Some(c) + && id != "comp.open" + { + match prev { + Some(p) if s.project.comp(p).is_some() => s.state.active_comp = Some(p), + _ => { + if !s.state.open_comps.contains(&c) { + s.state.open_comps.push(c); + } + } + } + } + r.map_err(|e| e.to_string()) +} + +/// `__query(kind, json)` → JSON. +fn native_query(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let kind = arg_string(args, 0, ctx)?; + let a = arg_json(args, 1, ctx)?; + let r = with_active(|act| match kind.as_str() { + "app" => Ok(json!({ + "version": format!("26.0 (EffectCraft {})", env!("CARGO_PKG_VERSION")), + "buildName": env!("CARGO_PKG_VERSION"), + "scriptName": act.name, + "allowFiles": act.session.prefs.scripting.allow_scripts_write_files, + "projectDir": project_dir(&act.session), + })), + _ => crate::model::query(&mut act.session, &kind, &a), + }); + match r { + Ok(v) => Ok(js_str(&v.to_string())), + Err(e) => Err(throw(e)), + } +} + +/// `__print(text)`: `writeLn`, `$.writeln`, `alert`. +fn native_print(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let text = arg_string(args, 0, ctx)?; + let _ = with_active(|a| { + a.output.extend(text.split('\n').map(str::to_string)); + Ok(()) + }); + Ok(JsValue::undefined()) +} + +/// `__undo("begin" | "end", name)`: After Effects' `app.beginUndoGroup` / `endUndoGroup`. Every +/// command run inside the outermost group folds into one undo step named after the group. +fn native_undo(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + let name = arg_string(args, 1, ctx)?; + with_active(|a| { + match op.as_str() { + "begin" => { + let snap = a.session.project.clone(); + a.groups.push((if name.is_empty() || name == "undefined" { "Script".into() } else { name }, snap)); + } + _ => { + if a.groups.len() == 1 { + let (name, snap) = a.groups.pop().unwrap_or_default(); + collapse(&mut a.session, &name, &snap); + } else { + a.groups.pop(); + } + } + } + Ok(()) + }) + .map_err(throw)?; + Ok(JsValue::undefined()) +} + +/// Fold the undo steps recorded since the project was `snap` into one step called `name`. +fn collapse(s: &mut Session, name: &str, snap: &Arc) { + let h = &mut s.history; + let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { + // Nothing was recorded (or history was reset by New / Open Project). + return; + }; + let first = h.undo[start].1.clone(); + h.undo.truncate(start); + h.undo.push((name.to_string(), first)); + h.merge_key = None; +} + +/// `__task("schedule", code, repeat)` → id; `__task("cancel", id)`. +fn native_task(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + if op == "cancel" { + let id = args.get(1).cloned().unwrap_or_default().to_number(ctx)? as u32; + let _ = with_active(|a| { + a.tasks.retain(|t| t.0 != id); + Ok(()) + }); + return Ok(JsValue::undefined()); + } + let code = arg_string(args, 1, ctx)?; + let repeat = args.get(2).cloned().unwrap_or_default().to_boolean(); + let id = with_active(|a| { + a.next_task += 1; + let id = a.next_task; + a.tasks.push((id, code, repeat)); + Ok(id) + }) + .map_err(throw)?; + Ok(JsValue::from(id)) +} + +fn project_dir(s: &Session) -> Option { + let p = s.path.as_deref()?; + std::path::Path::new(p).parent().map(|d| d.to_string_lossy().to_string()) +} + +/// Lexically normalized absolute path (`a/./b/../c` → `a/c`). +fn normalize(p: &str) -> std::path::PathBuf { + use std::path::{Component, Path, PathBuf}; + let path = Path::new(p); + let abs = if path.is_absolute() { path.to_path_buf() } else { std::env::current_dir().unwrap_or_default().join(path) }; + let mut out = PathBuf::new(); + for c in abs.components() { + match c { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + c => out.push(c), + } + } + // Resolve symlinks where the file (or its folder) exists, so links can't escape the project. + std::fs::canonicalize(&out) + .or_else(|_| out.parent().map(std::fs::canonicalize).unwrap_or(Ok(out.clone())).map(|d| d.join(out.file_name().unwrap_or_default()))) + .unwrap_or(out) +} + +const GATE: &str = "Preferences ▸ Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network"; + +/// The file-access gate: reads under the project's folder are always allowed; other reads, all +/// writes and listing other folders need the Allow Scripts to Write Files preference. +fn check_access(s: &Session, path: &str, write: bool) -> Result { + let p = normalize(path); + if s.prefs.scripting.allow_scripts_write_files { + return Ok(p); + } + if write { + return Err(format!("Unable to write {path}: scripts can't write files unless {GATE} is on")); + } + if let Some(dir) = project_dir(s) { + let d = normalize(&dir); + if p.starts_with(&d) { + return Ok(p); + } + } + Err(format!("Unable to access {path}: scripts can only read files in the project's folder unless {GATE} is on")) +} + +/// `__file(op, path, data)`: `read` → text, `write`/`append` → bytes, `exists` → bool, `list` → +/// names, `mkdir`, `remove`, `isDir`. +fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let op = arg_string(args, 0, ctx)?; + let path = arg_string(args, 1, ctx)?; + let data = if args.len() > 2 { arg_string(args, 2, ctx)? } else { String::new() }; + let r: Result = with_active(|a| { + let s = &a.session; + match op.as_str() { + "access" => Ok(json!(check_access(s, &path, false)?.exists())), + "exists" | "isDir" => { + // Unreadable locations report "missing" instead of failing. + let Ok(p) = check_access(s, &path, false) else { return Ok(json!(false)) }; + Ok(json!(if op == "exists" { p.exists() } else { p.is_dir() })) + } + "read" => { + let p = check_access(s, &path, false)?; + let bytes = s.services.read_file(&p.to_string_lossy()).map_err(|e| format!("Unable to read {path}: {e}"))?; + Ok(json!(String::from_utf8_lossy(&bytes))) + } + "write" | "append" => { + let p = check_access(s, &path, true)?; + let ps = p.to_string_lossy().to_string(); + let mut bytes = if op == "append" { s.services.read_file(&ps).unwrap_or_default() } else { vec![] }; + bytes.extend_from_slice(data.as_bytes()); + s.services.write_file(&ps, &bytes).map_err(|e| format!("Unable to write {path}: {e}"))?; + Ok(json!(bytes.len())) + } + "list" => { + let p = check_access(s, &path, false)?; + let mut names: Vec = std::fs::read_dir(&p) + .map_err(|e| format!("Unable to list {path}: {e}"))? + .filter_map(|e| e.ok()) + .map(|e| e.path().to_string_lossy().to_string()) + .collect(); + names.sort(); + Ok(json!(names)) + } + "mkdir" => { + let p = check_access(s, &path, true)?; + std::fs::create_dir_all(&p).map_err(|e| format!("Unable to create {path}: {e}"))?; + Ok(json!(true)) + } + "remove" => { + let p = check_access(s, &path, true)?; + let r = if p.is_dir() { std::fs::remove_dir(&p) } else { std::fs::remove_file(&p) }; + Ok(json!(r.is_ok())) + } + "network" => Err(if s.prefs.scripting.allow_scripts_write_files { + "network access from scripts is not supported in EffectCraft".to_string() + } else { + format!("scripts can't access the network unless {GATE} is on") + }), + _ => Err(format!("unknown file operation `{op}`")), + } + }); + match r { + Ok(v) => Ok(js_str(&v.to_string())), + Err(e) => Err(throw(e)), + } +} + +fn new_context() -> Result { + let mut ctx = Context::default(); + let limits = ctx.runtime_limits_mut(); + limits.set_loop_iteration_limit(200_000_000); + limits.set_recursion_limit(2000); + let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 6] = [ + (js_string!("__exec"), 2, native_exec), + (js_string!("__query"), 2, native_query), + (js_string!("__print"), 1, native_print), + (js_string!("__undo"), 2, native_undo), + (js_string!("__task"), 3, native_task), + (js_string!("__file"), 3, native_file), + ]; + for (name, len, f) in natives { + ctx.register_global_callable(name, len, NativeFunction::from_fn_ptr(f)).map_err(|e| e.to_string())?; + } + ctx.eval(Source::from_bytes(PRELUDE)).map_err(|e| format!("script prelude: {e}"))?; + Ok(ctx) +} + +/// Turn a boa error into a message with line and column (`code` is the user's script). +fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { + // Thrown values: `throw new Error("x")`, `throw "x"`, native errors. + let display = e.to_string(); + let message = match e.as_opaque() { + Some(v) => { + let o = v.as_object(); + let msg = o.as_ref().and_then(|o| o.get(js_string!("message"), ctx).ok()).filter(|m| !m.is_undefined()); + let name = o.as_ref().and_then(|o| o.get(js_string!("name"), ctx).ok()).filter(|m| !m.is_undefined()); + match (name, msg) { + (Some(n), Some(m)) => { + let n = n.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + let m = m.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + if n == "Error" { m } else { format!("{n}: {m}") } + } + _ => v.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_else(|_| display.clone()), + } + } + None => { + let first = display.lines().next().unwrap_or("").trim(); + let first = first.split(" at line ").next().unwrap_or(first).trim(); + let first = match first.rfind(" (") { + Some(i) if first.ends_with(')') && first[i..].contains(':') => first[..i].trim(), + _ => first, + }; + first.strip_prefix("Error: ").unwrap_or(first).to_string() + } + }; + let (line, column) = position(&display, file, code); + ScriptError { message, line, column, file: file.to_string() } +} + +/// Line and column of an error: `at line N, col M` (syntax errors), else the innermost backtrace +/// frame in the user's script (`file:line:col`), else (names boa can't place) the first use of +/// the undefined name in `code`. +fn position(msg: &str, file: &str, code: &str) -> (Option, Option) { + if let Some(i) = msg.find("at line ") { + let rest = &msg[i + 8..]; + let line = rest.split(|c: char| !c.is_ascii_digit()).next().and_then(|n| n.parse().ok()); + let col = rest.find("col ").and_then(|j| rest[j + 4..].split(|c: char| !c.is_ascii_digit()).next()).and_then(|n| n.parse().ok()); + return (line, col); + } + // `(file:12:5)`: the digits after the file name. + let pick = |l: &str| -> Option<(u32, u32)> { + let i = l.find(&format!("({file}:"))? + file.len() + 2; + let mut it = l[i..].split([':', ')']); + Some((it.next()?.parse().ok()?, it.next()?.parse().ok()?)) + }; + let frames: Vec<&str> = msg.lines().skip(1).collect(); + if let Some((a, b)) = frames.iter().find_map(|l| pick(l)).or_else(|| msg.lines().next().and_then(pick)) { + return (Some(a), Some(b)); + } + // `ReferenceError: name is not defined` without a position. + let first = msg.lines().next().unwrap_or(""); + if let Some(name) = first.split("ReferenceError: ").nth(1).and_then(|r| r.split(" is not defined").next()) { + for (n, line) in code.lines().enumerate() { + let b = line.as_bytes(); + let mut from = 0; + while let Some(i) = line[from..].find(name).map(|i| i + from) { + let before = i.checked_sub(1).map(|j| b[j]); + let after = b.get(i + name.len()).copied(); + let ident = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; + if !before.is_some_and(|c| ident(c) || c == b'.') && !after.is_some_and(ident) { + return (Some(n as u32 + 1), Some(i as u32 + 1)); + } + from = i + name.len(); + } + } + } + (None, None) +} + +/// Evaluate `code` in `ctx` and convert the result. +fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { + let src = Source::from_bytes(code.as_bytes()).with_path(std::path::Path::new(file)); + let script = boa_engine::Script::parse(src, None, ctx).map_err(|e| script_error(&e, ctx, file, code))?; + let v = script.evaluate(ctx).map_err(|e| script_error(&e, ctx, file, code))?; + Ok(result_json(&v, ctx)) +} + +/// The script's completion value as JSON (`__result` in the prelude does the conversion). +fn result_json(v: &JsValue, ctx: &mut Context) -> J { + if v.is_undefined() { + return J::Null; + } + let global = ctx.global_object(); + let Some(f) = global.get(js_string!("__result"), ctx).ok().and_then(|f| f.as_object()) else { return J::Null }; + match f.call(&JsValue::undefined(), std::slice::from_ref(v), ctx) { + Ok(s) if s.is_string() => { + let s = s.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + serde_json::from_str(&s).unwrap_or(J::String(s)) + } + _ => J::Null, + } +} + +/// Restores the caller's session when a run ends (on every exit path). +struct Restore<'a> { + target: &'a mut Session, +} + +impl Drop for Restore<'_> { + fn drop(&mut self) { + if let Some(a) = ACTIVE.with(|c| c.borrow_mut().take()) { + *self.target = a.session; + } + } +} + +/// Run a script against `session`: the object model sees and edits that session through engine +/// commands. Script errors come back in [`Outcome::error`]. +pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { + let running = ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)); + if running { + return Outcome { + error: Some(ScriptError { message: "scripts can't run scripts".into(), file: req.name.into(), ..Default::default() }), + ..Default::default() + }; + } + let s = std::mem::take(session); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active { session: s, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: req.name.to_string() })); + let guard = Restore { target: session }; + let file = if req.name.is_empty() { "script" } else { req.name }; + let mut result = if req.console { + CONSOLE.with(|c| { + let mut c = c.borrow_mut(); + if c.is_none() { + match new_context() { + Ok(ctx) => *c = Some(ManuallyDrop::new(ctx)), + Err(e) => return Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + } + } + let ctx = c.as_mut().map(|m| &mut **m).expect("console context"); + eval_in(ctx, req.code, file) + }) + } else { + match new_context() { + Ok(mut ctx) => { + let r = eval_in(&mut ctx, req.code, file); + // app.scheduleTask: run the queued one-shot tasks after the script, in order. + let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); + let mut r = r; + for (_, code, repeat) in tasks { + if repeat { + continue; + } + if let Err(e) = eval_in(&mut ctx, &code, "scheduled task") + && r.is_ok() + { + r = Err(e); + } + } + r + } + Err(e) => Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + } + }; + // Close undo groups left open; drop selections that no longer exist. + let output = with_active(|a| { + if let Some((name, snap)) = a.groups.first().cloned() { + collapse(&mut a.session, &name, &snap); + a.groups.clear(); + } + a.session.sanitize_state(); + Ok(std::mem::take(&mut a.output)) + }) + .unwrap_or_default(); + drop(guard); + let mut out = Outcome { output, ..Default::default() }; + match result.as_mut() { + Ok(v) => out.result = std::mem::take(v), + Err(e) => out.error = Some(e.clone()), + } + out +} diff --git a/crates/script/src/tests.rs b/crates/script/src/tests.rs new file mode 100644 index 000000000..458f2b41d --- /dev/null +++ b/crates/script/src/tests.rs @@ -0,0 +1,448 @@ +//! End-to-end scripts against a headless session: build with the object model, read back through +//! the engine. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::keyframe::Value as KV; +use effectcraft_engine::project::{Comp, ItemId, Layer}; +use serde_json::{Value, json}; + +use crate::{Outcome, run_code}; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), ..Default::default() }; + crate::install(&mut s); + s +} + +fn ok(s: &mut Session, code: &str) -> Outcome { + let o = run_code(s, code, "test.jsx"); + assert!(o.error.is_none(), "script failed: {:?}\noutput: {:?}", o.error, o.output); + o +} + +fn comp_named<'a>(s: &'a Session, name: &str) -> (ItemId, &'a Comp) { + let it = s.project.items.values().find(|i| i.name == name && i.as_comp().is_some()).unwrap_or_else(|| panic!("no comp {name}")); + (it.id, it.as_comp().unwrap()) +} + +fn layer<'a>(c: &'a Comp, name: &str) -> &'a Layer { + c.layers.iter().find(|l| l.name == name).unwrap_or_else(|| panic!("no layer {name}")) +} + +#[test] +fn builds_a_comp_end_to_end() { + let mut s = session(); + let o = ok( + &mut s, + r#" + app.beginUndoGroup("Build"); + var comp = app.project.items.addComp("Main", 640, 360, 1, 4, 25); + var bg = comp.layers.addSolid([0.2, 0.4, 0.6], "BG", 640, 360, 1); + var txt = comp.layers.addText("Hello"); + txt.name = "Title"; + var pos = txt.property("ADBE Transform Group").property("ADBE Position"); + pos.setValueAtTime(0, [100, 180]); + pos.setValueAtTime(2, [540, 180]); + txt.transform.opacity.expression = "50 + 25"; + var blur = bg.Effects.addProperty("ADBE Gaussian Blur 2"); + blur.property("Blurriness").setValue(12); + var nul = comp.layers.addNull(); + txt.setParentWithJump(nul); + bg.blendingMode = BlendingMode.MULTIPLY; + var rq = app.project.renderQueue.items.add(comp); + rq.outputModule(1).file = new File("/tmp/effectcraft-script-test/out.mp4"); + app.endUndoGroup(); + writeLn("layers: " + comp.numLayers); + [comp.numLayers, pos.numKeys, pos.keyValue(2)[0], txt.transform.opacity.value, comp.layer(1).name] + "#, + ); + assert_eq!(o.output, vec!["layers: 3"]); + assert_eq!(o.result, json!([3, 2, 540, 75, "Null 1"])); + let (cid, c) = comp_named(&s, "Main"); + assert_eq!((c.width, c.height), (640, 360)); + assert_eq!(c.duration.seconds(), 4.0); + // AE adds new layers at the top. + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Null 1", "Title", "BG"]); + let title = layer(c, "Title"); + let p = title.props.prop("transform/position").unwrap(); + assert_eq!(p.keys.len(), 2); + assert_eq!(p.keys[1].value, KV::Vec3([540.0, 180.0, 0.0])); + assert_eq!(title.props.prop("transform/opacity").unwrap().expr.as_ref().unwrap().text, "50 + 25"); + assert_eq!(title.parent, Some(layer(c, "Null 1").id)); + let bg = layer(c, "BG"); + assert_eq!(bg.blend_mode, effectcraft_engine::color::BlendMode::Multiply); + let fx = bg.props.group("effects/#1").unwrap(); + assert_eq!(fx.match_id, "ec.blur.gaussian"); + assert_eq!(bg.props.prop("effects/#1/blurriness").unwrap().value, KV::Scalar(12.0)); + assert_eq!(s.project.render_queue.len(), 1); + assert_eq!(s.project.render_queue[0].comp, cid); + // One undo step for the whole group. + assert_eq!(s.history.undo.len(), 1); + assert_eq!(s.history.undo[0].0, "Build"); + assert!(s.undo()); + assert!(s.project.items.is_empty()); +} + +#[test] +fn undo_steps_without_a_group() { + let mut s = session(); + ok(&mut s, r#"var c = app.project.items.addComp("A", 100, 100, 1, 1, 24); c.layers.addNull(); c.layers.addNull();"#); + assert_eq!(s.history.undo.len(), 3); + // An unbalanced group is closed when the script ends. + ok(&mut s, r#"app.beginUndoGroup("Two"); var c = app.project.item(1); c.layers.addNull(); c.layers.addNull();"#); + assert_eq!(s.history.undo.len(), 4); + assert_eq!(s.history.undo[3].0, "Two"); + // Nested groups fold into the outermost one. + ok( + &mut s, + r#"app.beginUndoGroup("Outer"); app.beginUndoGroup("Inner"); app.project.item(1).layers.addNull(); app.endUndoGroup(); app.project.item(1).layers.addNull(); app.endUndoGroup();"#, + ); + assert_eq!(s.history.undo.len(), 5); + assert_eq!(s.history.undo[4].0, "Outer"); + s.undo(); + assert_eq!(s.project.comp(ItemId(1)).unwrap().layers.len(), 4); +} + +#[test] +fn errors_report_line_and_column() { + let mut s = session(); + let o = run_code(&mut s, "var a = 1;\nvar b = 2;\nfoo.bar();\n", "broken.jsx"); + let e = o.error.expect("error"); + assert_eq!(e.line, Some(3), "{e:?}"); + assert!(e.message.contains("foo"), "{e:?}"); + assert_eq!(e.file, "broken.jsx"); + let o = run_code(&mut s, "var a = 1;\n\nvar = ;\n", "syntax.jsx"); + let e = o.error.expect("syntax error"); + assert_eq!(e.line, Some(3), "{e:?}"); + let o = run_code(&mut s, "writeLn('before');\nthrow new Error('custom failure');\n", "throw.jsx"); + let e = o.error.expect("thrown"); + assert_eq!(e.message, "custom failure"); + assert_eq!(e.line, Some(2), "{e:?}"); + assert_eq!(o.output, vec!["before"]); + // Errors from the object model point at the user's line. + let o = run_code(&mut s, "var c = app.project.items.addComp('C', 10, 10, 1, 1, 24);\n\nc.layer(5);\n", "range.jsx"); + let e = o.error.expect("range error"); + assert!(e.message.contains("out of range"), "{e:?}"); + assert_eq!(e.line, Some(3), "{e:?}"); +} + +#[test] +fn match_names_and_lookups() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("M", 200, 200, 1, 2, 30); + var l = c.layers.addSolid([1, 1, 1], "S", 200, 200, 1); + var tr = l.property("ADBE Transform Group"); + var fx = l.property("ADBE Effect Parade").addProperty("Gaussian Blur"); + var m = l.property("ADBE Mask Parade").addProperty("ADBE Mask Atom"); + var sh = c.layers.addShape(); + var grp = sh.property("ADBE Root Vectors Group").addProperty("ADBE Vector Group"); + var rect = grp.property("ADBE Vectors Group").addProperty("ADBE Vector Shape - Rect"); + var fill = grp.property("Contents").addProperty("ADBE Vector Graphic - Fill"); + fill.property("ADBE Vector Fill Color").setValue([1, 0, 0]); + rect.property("ADBE Vector Rect Size").setValue([50, 60]); + [ + l.matchName, tr.matchName, tr.property("ADBE Position").matchName, tr.property("Anchor Point").matchName, + tr.property(1).name, tr.anchorPoint.matchName, l.transform.position.propertyValueType == PropertyValueType.ThreeD_SPATIAL, + fx.matchName, fx.property(1).matchName, fx.property("ADBE Gaussian Blur 2-0001").name, m.matchName, + m.property("ADBE Mask Shape").name, sh.matchName, grp.matchName, rect.matchName, fill.matchName, + tr.propertyType == PropertyType.NAMED_GROUP, l.Effects.propertyType == PropertyType.INDEXED_GROUP, + tr.position.parentProperty.matchName, tr.position.propertyDepth, l.property("Nope") === null, + fx.isEffect, m.isMask, l.Effects.numProperties + ] + "#, + ); + assert_eq!( + o.result, + json!([ + "ADBE AV Layer", + "ADBE Transform Group", + "ADBE Position", + "ADBE Anchor Point", + "Anchor Point", + "ADBE Anchor Point", + true, + "ADBE Gaussian Blur 2", + "ADBE Gaussian Blur 2-0001", + "Blurriness", + "ADBE Mask Atom", + "Mask Path", + "ADBE Vector Layer", + "ADBE Vector Group", + "ADBE Vector Shape - Rect", + "ADBE Vector Graphic - Fill", + true, + true, + "ADBE Transform Group", + 2, + true, + true, + true, + 1 + ]) + ); + let (_, c) = comp_named(&s, "M"); + let sh = c.layers.iter().find(|l| l.props.sub("contents").is_some()).unwrap(); + assert_eq!(sh.props.prop("contents/#1/contents/rect/size").unwrap().value, KV::Vec2([50.0, 60.0])); + assert_eq!(sh.props.prop("contents/#1/contents/fill/color").unwrap().value, KV::Color([1.0, 0.0, 0.0, 1.0])); +} + +#[test] +fn keyframes_eases_and_interpolation() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("K", 100, 100, 1, 5, 25); + var l = c.layers.addNull(); + var r = l.transform.rotation; + r.setValuesAtTimes([0, 1, 2], [0, 90, 180]); + r.setInterpolationTypeAtKey(2, KeyframeInterpolationType.HOLD, KeyframeInterpolationType.LINEAR); + r.setTemporalEaseAtKey(3, [new KeyframeEase(0, 75)]); + var idx = r.addKey(1.5); + r.removeKey(1); + var out = [r.numKeys, idx, r.keyTime(1), r.keyInInterpolationType(1) == KeyframeInterpolationType.HOLD, + r.keyOutInterpolationType(1) == KeyframeInterpolationType.LINEAR, r.keyInTemporalEase(3)[0].influence, + r.valueAtTime(1, false), r.isTimeVarying, r.nearestKeyIndex(1.9)]; + l.startTime = 1; + out.push(r.keyTime(1)); + out + "#, + ); + let v = o.result.as_array().unwrap(); + assert_eq!(v[0], json!(3)); + assert_eq!(v[1], json!(3)); + assert_eq!(v[2], json!(1)); + assert_eq!(v[3], json!(true)); + assert_eq!(v[4], json!(true)); + assert!((v[5].as_f64().unwrap() - 75.0).abs() < 1e-6, "{v:?}"); + assert_eq!(v[6], json!(90)); + assert_eq!(v[7], json!(true)); + assert_eq!(v[8], json!(3)); + // Moving the layer moves its keys (layer time): key 1 now sits at 2 s comp time. + assert_eq!(v[9], json!(2)); +} + +#[test] +fn text_documents() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("T", 400, 200, 1, 2, 30); + var t = c.layers.addText("abc"); + var st = t.property("ADBE Text Properties").property("ADBE Text Document"); + var doc = st.value; + doc.text = "Changed"; + doc.fontSize = 40; + doc.fillColor = [1, 0, 0]; + doc.justification = ParagraphJustification.CENTER_JUSTIFY; + doc.applyStroke = true; + doc.strokeWidth = 3; + st.setValue(doc); + var d2 = t.text.sourceText.value; + [d2.text, d2.fontSize, d2.fillColor, d2.justification == ParagraphJustification.CENTER_JUSTIFY, d2.strokeWidth, t instanceof TextLayer, t.name] + "#, + ); + assert_eq!(o.result, json!(["Changed", 40, [1, 0, 0], true, 3, true, "abc"])); + let (_, c) = comp_named(&s, "T"); + let KV::Text(d) = &c.layers[0].props.prop("text/sourceText").unwrap().value else { panic!() }; + assert_eq!(d.text, "Changed"); + assert_eq!(d.style_at(0).size, 40.0); +} + +#[test] +fn layers_items_and_markers() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var f = app.project.items.addFolder("Comps"); + var c = f.items.addComp("Inner", 100, 100, 1, 3, 30); + var main = app.project.items.addComp("Outer", 200, 200, 1, 6, 30); + var nested = main.layers.add(c); + var a = main.layers.addSolid([1, 0, 0], "A", 50, 50, 1, 2); + var b = main.layers.addSolid([0, 1, 0], "B", 50, 50, 1); + b.moveAfter(nested); + a.enabled = false; a.solo = true; a.locked = true; a.shy = true; + a.threeDLayer = true; + var cam = main.layers.addCamera("Cam", [100, 100]); + var lt = main.layers.addLight("Key", [100, 100]); + lt.lightType = LightType.SPOT; + var d = a.duplicate(); + d.name = "A copy"; + var mv = new MarkerValue("hello"); + mv.duration = 0.5; + main.markerProperty.setValueAtTime(1, mv); + b.marker.setValueAtTime(0.5, new MarkerValue("layer marker")); + b.inPoint = 1; b.outPoint = 3; + var pc = main.layers.precompose([b.index], "Pre", true); + [c.parentFolder.name, nested.source.name, a.outPoint, a.enabled, a.solo, a.locked, a.shy, a.threeDLayer, + cam instanceof CameraLayer, lt.lightType == LightType.SPOT, main.markerProperty.numKeys, main.markerProperty.keyValue(1).comment, + main.markerProperty.keyValue(1).duration, pc.name, pc instanceof CompItem, main.layer("A copy").index, app.project.numItems, + main.layers.byName("Pre") !== null, f.numItems] + "#, + ); + assert_eq!(o.result, json!(["Comps", "Inner", 2, false, true, true, true, true, true, true, 1, "hello", 0.5, "Pre", true, 3, 7, true, 1])); + let (_, c) = comp_named(&s, "Outer"); + assert_eq!(c.markers.len(), 1); + assert!( + c.layers + .iter() + .any(|l| l.name == "Key" + && matches!(l.source, effectcraft_engine::project::LayerSource::Light { kind: effectcraft_engine::project::LightKind::Spot })) + ); +} + +#[test] +fn security_gate_for_files() { + let dir = std::env::temp_dir().join(format!("ec-script-gate-{}", std::process::id())); + let proj = dir.join("proj"); + std::fs::create_dir_all(&proj).unwrap(); + std::fs::write(proj.join("data.txt"), "inside").unwrap(); + std::fs::write(dir.join("secret.txt"), "outside").unwrap(); + let mut s = session(); + s.path = Some(proj.join("p.ecproj").to_string_lossy().to_string()); + let q = |p: &std::path::Path| serde_json::to_string(&p.to_string_lossy()).unwrap(); + // Reading inside the project folder works. + let o = ok(&mut s, &format!("var f = new File({}); f.open('r'); var t = f.read(); f.close(); t", q(&proj.join("data.txt")))); + assert_eq!(o.result, json!("inside")); + // Outside it (also via `..`) is refused. + for path in [dir.join("secret.txt"), proj.join("../secret.txt")] { + let o = run_code(&mut s, &format!("var f = new File({}); f.open('r');", q(&path)), "gate.jsx"); + let e = o.error.expect("blocked"); + assert!(e.message.contains("Allow Scripts to Write Files"), "{e:?}"); + } + // Writing is refused anywhere, the network too. + let o = run_code(&mut s, &format!("var f = new File({}); f.open('w'); f.write('x'); f.close();", q(&proj.join("out.txt"))), "gate.jsx"); + assert!(o.error.is_some()); + assert!(!proj.join("out.txt").exists()); + assert!(run_code(&mut s, "new Socket()", "net.jsx").error.is_some()); + assert!(run_code(&mut s, "system.callSystem('ls')", "sys.jsx").error.is_some()); + // With the preference on, scripts can write and read anywhere. + s.prefs.scripting.allow_scripts_write_files = true; + ok(&mut s, &format!("var f = new File({}); f.open('w'); f.writeln('written'); f.close();", q(&dir.join("w.txt")))); + assert_eq!(std::fs::read_to_string(dir.join("w.txt")).unwrap(), "written\n"); + let o = ok(&mut s, &format!("var f = new File({}); f.open('r'); f.readln()", q(&dir.join("secret.txt")))); + assert_eq!(o.result, json!("outside")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn engine_entry_points() { + let mut s = session(); + // script.run reports errors in the result, not as a command failure. + let r = s.execute("script.run", json!({"code": "writeLn('hi'); 1 + 2"})).unwrap(); + assert_eq!(r["ok"], json!(true)); + assert_eq!(r["result"], json!(3)); + assert_eq!(r["output"], json!("hi")); + let r = s.execute("script.run", json!({"code": "\nnope()"})).unwrap(); + assert_eq!(r["ok"], json!(false)); + assert_eq!(r["error"]["line"], json!(2)); + // Scripts can't run scripts. + let r = s.execute("script.run", json!({"code": "app.run('script.run', {code: '1'})"})).unwrap(); + assert!(r["error"]["message"].as_str().unwrap().contains("scripts can't run scripts"), "{r}"); + // File ▸ Scripts ▸ Run Script File… runs .jsx as JavaScript. + let dir = std::env::temp_dir().join(format!("ec-script-file-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("make.jsx"); + std::fs::write(&f, "app.project.items.addComp('From File', 320, 240, 1, 2, 24);").unwrap(); + s.execute("file.runScript", json!({"path": f.to_string_lossy()})).unwrap(); + assert!(s.project.items.values().any(|i| i.name == "From File")); + std::fs::write(&f, "\n\nbroken(;").unwrap(); + let e = s.execute("file.runScript", json!({"path": f.to_string_lossy()})).unwrap_err().to_string(); + assert!(e.contains("make.jsx (line 3)"), "{e}"); + // JSON command scripts still work. + let j = dir.join("steps.json"); + std::fs::write(&j, r#"[{"command": "comp.new", "params": {"name": "From Steps"}}]"#).unwrap(); + s.execute("file.runScript", json!({"path": j.to_string_lossy()})).unwrap(); + assert!(s.project.items.values().any(|i| i.name == "From Steps")); + // Menu commands by name. + let o = ok(&mut s, "app.findMenuCommandId('New Project') > 0 && app.findMenuCommandId('No Such Thing') === 0"); + assert_eq!(o.result, json!(true)); + let o = ok(&mut s, "app.executeCommand(app.findMenuCommandId('New Project')); app.project.numItems"); + assert_eq!(o.result, json!(0)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn console_keeps_variables() { + let mut s = session(); + let req = |code: &'static str| effectcraft_engine::ScriptRequest { code, name: "console", console: true }; + let r = crate::run(&mut s, &req("var counter = 41;")); + assert!(r.error.is_none(), "{r:?}"); + let r = crate::run(&mut s, &req("counter + 1")); + assert_eq!(r.result, json!(42)); + let _ = Value::Null; +} + +#[test] +fn schedule_task_runs_after_the_script() { + let mut s = session(); + let o = ok( + &mut s, + "app.scheduleTask('writeLn(\"later\")', 100, false); var id = app.scheduleTask('writeLn(\"never\")', 10, false); app.cancelTask(id); writeLn('now');", + ); + assert_eq!(o.output, vec!["now", "later"]); +} + +#[test] +fn layer_switches_timing_and_comp_settings() { + let mut s = session(); + let o = ok( + &mut s, + r#" + var c = app.project.items.addComp("S", 320, 240, 1, 5, 30); + var matte = c.layers.addText("M"); + var fill = c.layers.addSolid([1, 1, 1], "Fill", 320, 240, 1); + fill.moveToEnd(); + fill.setTrackMatte(matte, TrackMatteType.LUMA); + var t1 = fill.trackMatteType == TrackMatteType.LUMA && fill.hasTrackMatte && fill.trackMatteLayer.name == "M"; + fill.stretch = 200; + fill.label = 3; + c.width = 400; + c.duration = 8; + c.bgColor = [0.1, 0.2, 0.3]; + c.time = 2; + c.workAreaStart = 1; + c.workAreaDuration = 3; + var extra = c.layers.addNull(); + extra.remove(); + fill.selected = true; + var sel = c.selectedLayers.length; + fill.transform.opacity.expression = "thisLayer.nope("; + var err = fill.transform.opacity.expressionError; + fill.transform.opacity.expressionEnabled = false; + var copy = c.duplicate(); + var f = app.project.items.addFolder("Bin"); + copy.parentFolder = f; + [t1, fill.stretch, fill.label, c.width, c.duration, c.bgColor[2], c.time, c.workAreaStart, c.workAreaDuration, + c.numLayers, sel, err.length > 0, fill.transform.opacity.expressionEnabled, copy.name, f.numItems, copy.parentFolder.name] + "#, + ); + let v = o.result; + assert_eq!(v[0], json!(true), "{v}"); + assert_eq!(v[1], json!(200)); + assert_eq!(v[2], json!(3)); + assert_eq!(v[3], json!(400)); + assert_eq!(v[4], json!(8)); + assert!((v[5].as_f64().unwrap() - 0.3).abs() < 1e-6); + assert_eq!(v[6], json!(2)); + assert_eq!(v[7], json!(1)); + assert_eq!(v[8], json!(3)); + assert_eq!(v[9], json!(2)); + assert_eq!(v[10], json!(1)); + assert_eq!(v[11], json!(true)); + assert_eq!(v[12], json!(false)); + assert_eq!(v[13], json!("S 2")); + assert_eq!(v[14], json!(1)); + assert_eq!(v[15], json!("Bin")); + // Removing the copy through the object model. + let o = ok(&mut s, "var n = app.project.numItems; app.project.item(1).remove(); n - app.project.numItems"); + assert_eq!(o.result, json!(1)); +} diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml new file mode 100644 index 000000000..0e1af7719 --- /dev/null +++ b/crates/svg/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "effectcraft-svg" +description = "SVG import: parses SVG 1.1 / SVG 2 static vector content (paths, basic shapes, groups, transforms, use, CSS styles, solid and gradient fills/strokes, opacity) and rasterises it at any scale" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-path = { workspace = true } +kurbo = { workspace = true } +roxmltree = "0.21" +rayon = { workspace = true } diff --git a/crates/svg/README.md b/crates/svg/README.md new file mode 100644 index 000000000..862bd01d0 --- /dev/null +++ b/crates/svg/README.md @@ -0,0 +1,26 @@ +# effectcraft-svg + +SVG import for EffectCraft: `.svg` files import as footage (rasterised at any scale, so +Continuously Rasterize stays sharp) and Layer ▸ Create ▸ Create Shapes from Vector Layer turns +them into editable shape layers. + +Implemented from the W3C **Scalable Vector Graphics (SVG) 1.1 (Second Edition)** recommendation +(16 August 2011) and the **SVG 2** candidate recommendation (4 October 2018) for the parts listed +here, plus **CSS Color Module Level 3** for colour keywords and functions: + +- document: `svg` (width/height, `viewBox`, `preserveAspectRatio`), nested `svg`, `g`, `defs`, + `use` (x/y, `href` / `xlink:href`), `symbol` (through `use`), `title`/`desc`/`metadata` ignored; +- shapes: `path` (all path data commands incl. elliptical arcs), `rect` (rx/ry), `circle`, + `ellipse`, `line`, `polyline`, `polygon`; +- `transform` lists (matrix, translate, scale, rotate with centre, skewX, skewY); +- painting: `fill`, `fill-rule`, `fill-opacity`, `stroke`, `stroke-width`, `stroke-linecap`, + `stroke-linejoin`, `stroke-miterlimit`, `stroke-dasharray`, `stroke-dashoffset`, + `stroke-opacity`, `opacity`, `display`, `visibility`, `color`/`currentColor`; +- `linearGradient` and `radialGradient` (with `fx`/`fy` focal point), `gradientUnits`, + `gradientTransform`, `spreadMethod`, `href` inheritance of stops and attributes; +- styling through presentation attributes, the `style` attribute and ` + + + + + + + + + + + + + + + + + diff --git a/crates/svg/tests/fixtures/paths-gradients.svg.attribution b/crates/svg/tests/fixtures/paths-gradients.svg.attribution new file mode 100644 index 000000000..d4b8758fe --- /dev/null +++ b/crates/svg/tests/fixtures/paths-gradients.svg.attribution @@ -0,0 +1,7 @@ +asset: paths-gradients.svg +title: SVG test fixture: path data, gradients, CSS styles and use +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import) +notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests. diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92b..33ffacae1 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization, Roto Brush segmentation (graph cut, matting) and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 000000000..72cb2f174 --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,470 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + tries += 1; + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; + if !o.fix_points { + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } + } + } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-6 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 000000000..30b019e54 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 000000000..6956ac2b5 --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 000000000..b719cc8e5 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,486 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 000000000..76c0637f6 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,380 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 000000000..85edd2bd8 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,953 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise() + } + fn huber(&self) -> f64 { + self.noise() + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise() / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + lap("tripod"); + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); + if cancelled() { + return err("cancelled"); + } + lap("keyframe adjustment"); + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + lap("resection"); + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + lap("final adjustment"); + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 000000000..4b5a83460 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,438 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5f..1c6e6a156 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,10 +18,12 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; pub mod plane; +pub mod roto; pub mod solve; pub mod stabilize; diff --git a/crates/track/src/roto/gmm.rs b/crates/track/src/roto/gmm.rs new file mode 100644 index 000000000..c0fe95b05 --- /dev/null +++ b/crates/track/src/roto/gmm.rs @@ -0,0 +1,228 @@ +//! Gaussian mixture colour models (full 3 × 3 covariances), as used by GrabCut (C. Rother, +//! V. Kolmogorov and A. Blake, "GrabCut: Interactive Foreground Extraction using Iterated Graph +//! Cuts", SIGGRAPH 2004). +//! +//! Fitting is deterministic: components are initialised by recursive splitting of the cluster +//! with the largest variance along its principal axis (Orchard & Bouman, "Color Quantization of +//! Images", IEEE TSP 1991), then refined by a few hard-assignment EM iterations. + +use serde::{Deserialize, Serialize}; + +/// One Gaussian: weight, mean, inverse covariance and log normaliser. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Gauss { + pub weight: f64, + pub mean: [f64; 3], + pub inv: [[f64; 3]; 3], + /// −log(weight) + ½·log(det Σ) (the 2π term is dropped: it cancels between models). + pub log_norm: f64, +} + +/// A colour model. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Gmm { + pub comps: Vec, +} + +/// Regularisation added to every covariance's diagonal (colour noise floor). +const REG: f64 = 2e-4; + +fn det3(m: &[[f64; 3]; 3]) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} + +fn inv3(m: &[[f64; 3]; 3]) -> Option<[[f64; 3]; 3]> { + let d = det3(m); + if d.abs() < 1e-30 { + return None; + } + let mut r = [[0.0; 3]; 3]; + r[0][0] = (m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d; + r[0][1] = (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d; + r[0][2] = (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d; + r[1][0] = (m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d; + r[1][1] = (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d; + r[1][2] = (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d; + r[2][0] = (m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d; + r[2][1] = (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d; + r[2][2] = (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d; + Some(r) +} + +/// Sufficient statistics of a cluster. +#[derive(Clone, Copy, Default)] +struct Stats { + n: f64, + s: [f64; 3], + ss: [[f64; 3]; 3], +} + +impl Stats { + fn add(&mut self, c: [f32; 3]) { + let c = [c[0] as f64, c[1] as f64, c[2] as f64]; + self.n += 1.0; + for i in 0..3 { + self.s[i] += c[i]; + for j in 0..3 { + self.ss[i][j] += c[i] * c[j]; + } + } + } + fn mean(&self) -> [f64; 3] { + let n = self.n.max(1.0); + [self.s[0] / n, self.s[1] / n, self.s[2] / n] + } + fn cov(&self) -> [[f64; 3]; 3] { + let n = self.n.max(1.0); + let m = self.mean(); + let mut c = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + c[i][j] = self.ss[i][j] / n - m[i] * m[j]; + } + c[i][i] += REG; + } + c + } +} + +/// Principal eigenvector and eigenvalue of a symmetric 3 × 3 matrix (power iteration). +fn principal(c: &[[f64; 3]; 3]) -> ([f64; 3], f64) { + let mut v = [0.577, 0.577, 0.577]; + let mut lambda = 0.0; + for _ in 0..40 { + let w = [ + c[0][0] * v[0] + c[0][1] * v[1] + c[0][2] * v[2], + c[1][0] * v[0] + c[1][1] * v[1] + c[1][2] * v[2], + c[2][0] * v[0] + c[2][1] * v[1] + c[2][2] * v[2], + ]; + let n = (w[0] * w[0] + w[1] * w[1] + w[2] * w[2]).sqrt(); + if n < 1e-30 { + break; + } + lambda = n; + v = [w[0] / n, w[1] / n, w[2] / n]; + } + (v, lambda) +} + +impl Gmm { + /// Fit `k` components to `samples` (at most `max_samples` are used, taken at a fixed stride). + pub fn fit(samples: &[[f32; 3]], k: usize, max_samples: usize) -> Gmm { + if samples.is_empty() { + return Gmm::default(); + } + let stride = samples.len().div_ceil(max_samples.max(1)).max(1); + let xs: Vec<[f32; 3]> = samples.iter().step_by(stride).copied().collect(); + // Orchard–Bouman splitting. + let mut label = vec![0usize; xs.len()]; + let mut ncl = 1; + while ncl < k { + let mut st = vec![Stats::default(); ncl]; + for (x, l) in xs.iter().zip(&label) { + st[*l].add(*x); + } + let (mut best, mut best_l, mut axis) = (usize::MAX, 0.0, [0.0; 3]); + for (c, s) in st.iter().enumerate() { + if s.n < 2.0 { + continue; + } + let (v, l) = principal(&s.cov()); + if l > best_l { + best = c; + best_l = l; + axis = v; + } + } + if best == usize::MAX || best_l <= REG * 1.01 { + break; + } + let m = st[best].mean(); + let thr = axis[0] * m[0] + axis[1] * m[1] + axis[2] * m[2]; + for (x, l) in xs.iter().zip(label.iter_mut()) { + if *l == best && axis[0] * x[0] as f64 + axis[1] * x[1] as f64 + axis[2] * x[2] as f64 > thr { + *l = ncl; + } + } + ncl += 1; + } + let mut g = Gmm::from_labels(&xs, &label, ncl); + // Hard EM. + for _ in 0..3 { + for (x, l) in xs.iter().zip(label.iter_mut()) { + *l = g.best(*x); + } + let ng = Gmm::from_labels(&xs, &label, g.comps.len()); + if ng.comps.is_empty() { + break; + } + g = ng; + } + g + } + + fn from_labels(xs: &[[f32; 3]], label: &[usize], k: usize) -> Gmm { + let mut st = vec![Stats::default(); k]; + for (x, l) in xs.iter().zip(label) { + st[*l].add(*x); + } + let total: f64 = st.iter().map(|s| s.n).sum::().max(1.0); + let comps = st + .iter() + .filter(|s| s.n >= 1.0) + .filter_map(|s| { + let c = s.cov(); + let inv = inv3(&c)?; + let det = det3(&c).max(1e-300); + let weight = s.n / total; + Some(Gauss { weight, mean: s.mean(), inv, log_norm: -weight.ln() + 0.5 * det.ln() }) + }) + .collect(); + Gmm { comps } + } + + #[inline] + fn comp_cost(g: &Gauss, c: [f32; 3]) -> f64 { + let d = [c[0] as f64 - g.mean[0], c[1] as f64 - g.mean[1], c[2] as f64 - g.mean[2]]; + let mut q = 0.0; + for i in 0..3 { + q += d[i] * (g.inv[i][0] * d[0] + g.inv[i][1] * d[1] + g.inv[i][2] * d[2]); + } + g.log_norm + 0.5 * q + } + + /// The most likely component of colour `c`. + pub fn best(&self, c: [f32; 3]) -> usize { + let mut b = 0; + let mut bv = f64::INFINITY; + for (i, g) in self.comps.iter().enumerate() { + let v = Self::comp_cost(g, c); + if v < bv { + bv = v; + b = i; + } + } + b + } + + /// −log p(c) (up to a constant shared by all models), clamped to a finite range. + pub fn cost(&self, c: [f32; 3]) -> f64 { + if self.comps.is_empty() { + return 10.0; + } + // log-sum-exp over components. + let mut m = f64::INFINITY; + let mut costs = [0.0f64; 16]; + let n = self.comps.len().min(16); + for (i, g) in self.comps.iter().take(n).enumerate() { + costs[i] = Self::comp_cost(g, c); + m = m.min(costs[i]); + } + let s: f64 = costs[..n].iter().map(|v| (m - v).exp()).sum(); + (m - s.ln()).clamp(-20.0, 60.0) + } + + pub fn is_empty(&self) -> bool { + self.comps.is_empty() + } +} diff --git a/crates/track/src/roto/matting.rs b/crates/track/src/roto/matting.rs new file mode 100644 index 000000000..439d9c43e --- /dev/null +++ b/crates/track/src/roto/matting.rs @@ -0,0 +1,465 @@ +//! Matte utilities: exact Euclidean distance transforms (Felzenszwalb & Huttenlocher, +//! "Distance Transforms of Sampled Functions", Theory of Computing 8, 2012), box and Gaussian +//! filters, the colour **guided filter** (K. He, J. Sun and X. Tang, "Guided Image Filtering", +//! ECCV 2010 / IEEE PAMI 2013) used for edge matting, foreground colour estimation +//! (decontamination, from the compositing equation I = αF + (1 − α)B with B and F estimated from +//! nearby pure background / foreground pixels), and the Refine Edge matte adjustments. + +use rayon::prelude::*; + +const BIG: f32 = 1e12; + +/// 1-D squared distance transform of `f` (lower envelope of parabolas). +fn dt1(f: &[f32], d: &mut [f32], v: &mut [usize], z: &mut [f64]) { + let n = f.len(); + if n == 0 { + return; + } + let ff = |q: usize| f[q] as f64 + (q * q) as f64; + let mut k = 0usize; + v[0] = 0; + z[0] = f64::NEG_INFINITY; + z[1] = f64::INFINITY; + for q in 1..n { + let mut s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + while s <= z[k] { + if k == 0 { + break; + } + k -= 1; + s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + } + k += 1; + v[k] = q; + z[k] = s; + z[k + 1] = f64::INFINITY; + } + k = 0; + for (q, dq) in d.iter_mut().enumerate().take(n) { + while z[k + 1] < q as f64 { + k += 1; + } + let p = v[k]; + let dd = q as f64 - p as f64; + *dq = (dd * dd + f[p] as f64) as f32; + } +} + +/// Euclidean distance from every pixel to the nearest pixel where `seed` is true (BIG.sqrt() +/// when there is none). +pub fn distance(seed: &[bool], w: usize, h: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + let mut g: Vec = seed.iter().map(|s| if *s { 0.0 } else { BIG }).collect(); + // Columns. + let cols: Vec> = (0..w) + .into_par_iter() + .map(|x| { + let f: Vec = (0..h).map(|y| g[y * w + x]).collect(); + let mut d = vec![0.0; h]; + let (mut v, mut z) = (vec![0usize; h], vec![0.0f64; h + 1]); + dt1(&f, &mut d, &mut v, &mut z); + d + }) + .collect(); + for (x, c) in cols.iter().enumerate() { + for y in 0..h { + g[y * w + x] = c[y]; + } + } + // Rows. + g.par_chunks_mut(w).for_each(|row| { + let f = row.to_vec(); + let (mut v, mut z) = (vec![0usize; w], vec![0.0f64; w + 1]); + dt1(&f, row, &mut v, &mut z); + }); + g.par_iter_mut().for_each(|v| *v = v.sqrt()); + g +} + +/// Signed distance to a binary matte's edge: positive inside (distance to the nearest +/// background pixel centre, minus ½), negative outside. +pub fn signed_distance(m: &[u8], w: usize, h: usize) -> Vec { + let fg: Vec = m.iter().map(|v| *v != 0).collect(); + let bg: Vec = m.iter().map(|v| *v == 0).collect(); + let din = distance(&bg, w, h); + let dout = distance(&fg, w, h); + m.iter().enumerate().map(|(i, v)| if *v != 0 { din[i] - 0.5 } else { 0.5 - dout[i] }).collect() +} + +/// Mean over the (2r+1)² window (clamped at the edges: averages only the pixels inside). +pub fn box_mean(src: &[f32], w: usize, h: usize, r: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + // Horizontal window sums per row. + let mut tmp = vec![0.0f64; w * h]; + tmp.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let s = &src[y * w..y * w + w]; + let mut acc = 0.0f64; + for x in 0..r.min(w) { + acc += s[x] as f64; + } + for (x, o) in row.iter_mut().enumerate() { + if x + r < w { + acc += s[x + r] as f64; + } + if x > r { + acc -= s[x - r - 1] as f64; + } + *o = acc; + } + }); + // Vertical prefix sums (row by row, cache friendly), then window differences. + let mut pre = vec![0.0f64; w * (h + 1)]; + for y in 0..h { + let (done, rest) = pre.split_at_mut((y + 1) * w); + let prev = &done[y * w..]; + let cur = &mut rest[..w]; + let t = &tmp[y * w..y * w + w]; + for x in 0..w { + cur[x] = prev[x] + t[x]; + } + } + let mut out = vec![0.0f32; w * h]; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let a = y.saturating_sub(r); + let b = (y + r + 1).min(h); + let cy = (b - a) as f64; + for (x, o) in row.iter_mut().enumerate() { + let cx = ((x + r + 1).min(w) - x.saturating_sub(r)) as f64; + *o = ((pre[b * w + x] - pre[a * w + x]) / (cx * cy)) as f32; + } + }); + out +} + +/// Approximate Gaussian blur (three box passes) with standard deviation `sigma` pixels. +pub fn gauss(src: &[f32], w: usize, h: usize, sigma: f64) -> Vec { + if sigma < 0.3 { + return src.to_vec(); + } + // Box radius whose three passes match the variance: 3·((2r+1)² − 1)/12 = σ². + let r = (((4.0 * sigma * sigma + 1.0).sqrt() - 1.0) / 2.0).round().max(1.0) as usize; + let a = box_mean(src, w, h, r); + let b = box_mean(&a, w, h, r); + box_mean(&b, w, h, r) +} + +fn inv3(m: [[f32; 3]; 3]) -> [[f32; 3]; 3] { + let d = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]); + let d = if d.abs() < 1e-20 { 1e-20 } else { d }; + [ + [(m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d, (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d, (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d], + [(m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d, (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d, (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d], + [(m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d, (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d, (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d], + ] +} + +/// The colour guided filter of `p` with guide `img` (RGB, w × h), window radius `r`, +/// regularisation `eps` (He et al. 2010, section 3.5). +pub fn guided_filter(img: &[[f32; 3]], p: &[f32], w: usize, h: usize, r: usize, eps: f32) -> Vec { + let n = w * h; + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let (ir, ig, ib) = (ch(0), ch(1), ch(2)); + let mul = |a: &[f32], b: &[f32]| -> Vec { a.iter().zip(b).map(|(x, y)| x * y).collect() }; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let (mr, mg, mb, mp) = (bm(&ir), bm(&ig), bm(&ib), bm(p)); + let (mrp, mgp, mbp) = (bm(&mul(&ir, p)), bm(&mul(&ig, p)), bm(&mul(&ib, p))); + let (vrr, vrg, vrb) = (bm(&mul(&ir, &ir)), bm(&mul(&ir, &ig)), bm(&mul(&ir, &ib))); + let (vgg, vgb, vbb) = (bm(&mul(&ig, &ig)), bm(&mul(&ig, &ib)), bm(&mul(&ib, &ib))); + let mut a = vec![[0.0f32; 3]; n]; + let mut b = vec![0.0f32; n]; + a.par_iter_mut().zip(b.par_iter_mut()).enumerate().for_each(|(i, (ai, bi))| { + let cov = [mrp[i] - mr[i] * mp[i], mgp[i] - mg[i] * mp[i], mbp[i] - mb[i] * mp[i]]; + let s = [ + [vrr[i] - mr[i] * mr[i] + eps, vrg[i] - mr[i] * mg[i], vrb[i] - mr[i] * mb[i]], + [vrg[i] - mr[i] * mg[i], vgg[i] - mg[i] * mg[i] + eps, vgb[i] - mg[i] * mb[i]], + [vrb[i] - mr[i] * mb[i], vgb[i] - mg[i] * mb[i], vbb[i] - mb[i] * mb[i] + eps], + ]; + let inv = inv3(s); + let av = [ + cov[0] * inv[0][0] + cov[1] * inv[1][0] + cov[2] * inv[2][0], + cov[0] * inv[0][1] + cov[1] * inv[1][1] + cov[2] * inv[2][1], + cov[0] * inv[0][2] + cov[1] * inv[1][2] + cov[2] * inv[2][2], + ]; + *ai = av; + *bi = mp[i] - av[0] * mr[i] - av[1] * mg[i] - av[2] * mb[i]; + }); + let (a0, a1, a2): (Vec, Vec, Vec) = (a.iter().map(|v| v[0]).collect(), a.iter().map(|v| v[1]).collect(), a.iter().map(|v| v[2]).collect()); + let (ma0, ma1, ma2, mbb) = (bm(&a0), bm(&a1), bm(&a2), bm(&b)); + (0..n).into_par_iter().map(|i| ma0[i] * ir[i] + ma1[i] * ig[i] + ma2[i] * ib[i] + mbb[i]).collect() +} + +/// Closed-form matting (A. Levin, D. Lischinski and Y. Weiss, "A Closed-Form Solution to Natural +/// Image Matting", IEEE PAMI 30(2), 2008): the alpha that minimises αᵀLα with the matting +/// Laplacian `L` of the colour image, with `known` pixels fixed to `init`. Solved by conjugate +/// gradients over the unknown pixels, starting from `init`; `L·x` is evaluated with box filters +/// (K. He, J. Sun and X. Tang, "Fast Matting Using Large Kernel Matting Laplacian Matrices", +/// CVPR 2010): `(L·x)ᵢ = Σ_{k∋i} (xᵢ − a_kᵀIᵢ − b_k)` with `(a_k, b_k)` the ridge-regression +/// fit of `x` to the colours in window k. +pub fn closed_form(img: &[[f32; 3]], init: &[f32], known: &[bool], w: usize, h: usize, r: usize, eps: f32, iters: usize) -> Vec { + let n = w * h; + if n == 0 || known.iter().all(|k| *k) { + return init.to_vec(); + } + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let ic = [ch(0), ch(1), ch(2)]; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let mu = [bm(&ic[0]), bm(&ic[1]), bm(&ic[2])]; + let prod = |a: usize, b: usize| -> Vec { ic[a].iter().zip(&ic[b]).map(|(x, y)| x * y).collect() }; + let mut cov = [[vec![], vec![], vec![]], [vec![], vec![], vec![]], [vec![], vec![], vec![]]]; + for a in 0..3 { + for b in a..3 { + let m = bm(&prod(a, b)); + cov[a][b] = m.clone(); + cov[b][a] = m; + } + } + let inv: Vec<[[f32; 3]; 3]> = (0..n) + .into_par_iter() + .map(|i| { + let mut s = [[0.0f32; 3]; 3]; + for a in 0..3 { + for b in 0..3 { + s[a][b] = cov[a][b][i] - mu[a][i] * mu[b][i] + if a == b { eps } else { 0.0 }; + } + } + inv3(s) + }) + .collect(); + // Pixels per window (= windows per pixel). + let cnt: Vec = { + let (rr, mut out) = (r as i64, vec![0.0f32; n]); + for (i, o) in out.iter_mut().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let cx = (x + rr).min(w as i64 - 1) - (x - rr).max(0) + 1; + let cy = (y + rr).min(h as i64 - 1) - (y - rr).max(0) + 1; + *o = (cx * cy) as f32; + } + out + }; + let lap = |x: &[f32]| -> Vec { + let pm = bm(x); + let ipm: Vec> = (0..3).map(|c| bm(&ic[c].iter().zip(x).map(|(a, b)| a * b).collect::>())).collect(); + let ab: Vec<[f32; 4]> = (0..n) + .into_par_iter() + .map(|i| { + let cv = [ipm[0][i] - mu[0][i] * pm[i], ipm[1][i] - mu[1][i] * pm[i], ipm[2][i] - mu[2][i] * pm[i]]; + let m = &inv[i]; + let a = [ + m[0][0] * cv[0] + m[0][1] * cv[1] + m[0][2] * cv[2], + m[1][0] * cv[0] + m[1][1] * cv[1] + m[1][2] * cv[2], + m[2][0] * cv[0] + m[2][1] * cv[1] + m[2][2] * cv[2], + ]; + [a[0], a[1], a[2], pm[i] - a[0] * mu[0][i] - a[1] * mu[1][i] - a[2] * mu[2][i]] + }) + .collect(); + let sa: Vec> = (0..3).map(|c| bm(&ab.iter().map(|v| v[c]).collect::>())).collect(); + let sb = bm(&ab.iter().map(|v| v[3]).collect::>()); + (0..n).into_par_iter().map(|i| cnt[i] * (x[i] - sa[0][i] * ic[0][i] - sa[1][i] * ic[1][i] - sa[2][i] * ic[2][i] - sb[i])).collect() + }; + // L_UU x = −(L α_K)_U. + let a0: Vec = (0..n).map(|i| if known[i] { init[i] } else { 0.0 }).collect(); + let la0 = lap(&a0); + let mut x: Vec = (0..n).map(|i| if known[i] { 0.0 } else { init[i] }).collect(); + let lx = lap(&x); + let mut res: Vec = (0..n).map(|i| if known[i] { 0.0 } else { (-la0[i] - lx[i]) as f64 }).collect(); + let mut p = res.clone(); + let mut rs: f64 = res.iter().map(|v| v * v).sum(); + let rs0 = rs.max(1e-30); + for _ in 0..iters { + if rs <= rs0 * 1e-8 { + break; + } + let pf: Vec = p.iter().map(|v| *v as f32).collect(); + let ap: Vec = lap(&pf).iter().enumerate().map(|(i, v)| if known[i] { 0.0 } else { *v as f64 }).collect(); + let pap: f64 = p.iter().zip(&ap).map(|(a, b)| a * b).sum(); + if pap <= 1e-30 { + break; + } + let alpha = rs / pap; + for i in 0..n { + x[i] += (alpha * p[i]) as f32; + res[i] -= alpha * ap[i]; + } + let rs_new: f64 = res.iter().map(|v| v * v).sum(); + let beta = rs_new / rs; + rs = rs_new; + for i in 0..n { + p[i] = res[i] + beta * p[i]; + } + } + (0..n).map(|i| if known[i] { init[i] } else { x[i].clamp(0.0, 1.0) }).collect() +} + +/// Contrast (0–100 %) around ½ on a soft alpha. +pub fn contrast(a: &mut [f32], pc: f64) { + if pc <= 0.0 { + return; + } + let k = 1.0 / (1.0 - 0.99 * (pc / 100.0).clamp(0.0, 1.0)) as f32; + a.par_iter_mut().for_each(|v| *v = (0.5 + (*v - 0.5) * k).clamp(0.0, 1.0)); +} + +/// Shift Edge (−100–100 %) on a soft alpha: positive values grow the matte (a gamma curve on α). +pub fn shift_soft(a: &mut [f32], pc: f64) { + if pc.abs() < 1e-9 { + return; + } + let g = 2f32.powf((-pc / 50.0) as f32); + a.par_iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0).powf(g)); +} + +/// Estimate the pure foreground colour of every pixel listed in `region` (alpha strictly +/// between 0 and 1) and mix it in by `amount` (0–1). `radius`: neighbourhood (pixels) that pure +/// background / foreground estimates are taken from. Returns the decontamination map (how much +/// each pixel's colour changed, 0–1). +pub fn decontaminate(img: &mut [[f32; 3]], alpha: &[f32], region: &[bool], w: usize, h: usize, radius: usize, amount: f32) -> Vec { + let n = w * h; + let mut map = vec![0.0f32; n]; + if !region.iter().any(|r| *r) || amount <= 0.0 { + return map; + } + let lo: Vec = alpha.iter().map(|a| if *a < 0.02 { 1.0 } else { 0.0 }).collect(); + let hi: Vec = alpha.iter().map(|a| if *a > 0.98 { 1.0 } else { 0.0 }).collect(); + // Windowed means of pure background / foreground colours, widening where none is near. + let est = |m: &[f32]| -> Vec> { + let mut out: Vec> = vec![None; n]; + let mut r = radius.max(2); + for _ in 0..4 { + let cnt = box_mean(m, w, h, r); + let sums: Vec> = (0..3).map(|c| box_mean(&img.iter().zip(m).map(|(p, k)| p[c] * k).collect::>(), w, h, r)).collect(); + let mut missing = false; + for i in 0..n { + if region[i] && out[i].is_none() { + if cnt[i] > 1e-4 { + out[i] = Some([sums[0][i] / cnt[i], sums[1][i] / cnt[i], sums[2][i] / cnt[i]]); + } else { + missing = true; + } + } + } + if !missing { + break; + } + r *= 2; + } + out + }; + let bgc = est(&lo); + let fgc = est(&hi); + for i in 0..n { + if !region[i] { + continue; + } + let a = alpha[i].clamp(0.0, 1.0); + if a <= 0.0 || a >= 1.0 { + continue; + } + let c = img[i]; + let f = match (bgc[i], fgc[i]) { + (Some(b), fbar) => { + let mut f = [0.0f32; 3]; + for k in 0..3 { + f[k] = ((c[k] - (1.0 - a) * b[k]) / a).clamp(0.0, 1.0); + } + // Low alphas amplify noise: lean on the nearby foreground colour there. + if let Some(fb) = fbar { + let t = ((0.25 - a) / 0.25).clamp(0.0, 1.0); + for k in 0..3 { + f[k] += (fb[k] - f[k]) * t; + } + } + f + } + (None, Some(fb)) => fb, + (None, None) => c, + }; + let mut d = 0.0f32; + for k in 0..3 { + let v = c[k] + (f[k] - c[k]) * amount; + d = d.max((v - c[k]).abs()); + img[i][k] = v; + } + map[i] = d.min(1.0); + } + map +} + +/// Sample a float plane bilinearly at continuous coordinates (pixel centres at +0.5), clamped. +pub fn sample(p: &[f32], w: usize, h: usize, x: f64, y: f64) -> f32 { + if w == 0 || h == 0 { + return 0.0; + } + let fx = (x - 0.5).clamp(0.0, (w - 1) as f64); + let fy = (y - 0.5).clamp(0.0, (h - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(w - 1), (y0 + 1).min(h - 1)); + let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); + let g = |x: usize, y: usize| p[y * w + x]; + let top = g(x0, y0) + (g(x1, y0) - g(x0, y0)) * tx; + let bot = g(x0, y1) + (g(x1, y1) - g(x0, y1)) * tx; + top + (bot - top) * ty +} + +/// Average of `a` sampled along the segment `[-v/2, v/2]` with `samples` taps (motion blur of a +/// matte moving by `v` pixels during the shutter). +pub fn motion_blur(a: &[f32], w: usize, h: usize, v: [f64; 2], samples: usize) -> Vec { + let n = samples.max(2); + if v[0].hypot(v[1]) < 0.25 { + return a.to_vec(); + } + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let mut s = 0.0; + for k in 0..n { + let t = k as f64 / (n - 1) as f64 - 0.5; + s += sample(a, w, h, x - v[0] * t, y - v[1] * t); + } + s / n as f32 + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn distance_matches_brute_force() { + let (w, h) = (23, 17); + let seed: Vec = (0..w * h).map(|i| (i * 7919) % 31 == 0).collect(); + let d = distance(&seed, w, h); + for y in 0..h { + for x in 0..w { + let mut b = f32::INFINITY; + for j in 0..h { + for i in 0..w { + if seed[j * w + i] { + b = b.min(((x as f32 - i as f32).powi(2) + (y as f32 - j as f32).powi(2)).sqrt()); + } + } + } + assert!((d[y * w + x] - b).abs() < 1e-3, "{x},{y}: {} vs {b}", d[y * w + x]); + } + } + } + + #[test] + fn guided_filter_recovers_a_linear_blend() { + // I = αF + (1 − α)B with constant F, B: the filtered binary matte is close to α. + let (w, h) = (64, 8); + let alpha: Vec = (0..w * h).map(|i| (((i % w) as f32 - 26.0) / 12.0).clamp(0.0, 1.0)).collect(); + let (f, b) = ([0.9f32, 0.2, 0.1], [0.1f32, 0.3, 0.8]); + let img: Vec<[f32; 3]> = alpha.iter().map(|a| [a * f[0] + (1.0 - a) * b[0], a * f[1] + (1.0 - a) * b[1], a * f[2] + (1.0 - a) * b[2]]).collect(); + let p: Vec = alpha.iter().map(|a| if *a >= 0.5 { 1.0 } else { 0.0 }).collect(); + let q = guided_filter(&img, &p, w, h, 8, 1e-5); + let err: f32 = q.iter().zip(&alpha).map(|(a, b)| (a.clamp(0.0, 1.0) - b).abs()).sum::() / (w * h) as f32; + assert!(err < 0.03, "mean error {err}"); + } +} diff --git a/crates/track/src/roto/maxflow.rs b/crates/track/src/roto/maxflow.rs new file mode 100644 index 000000000..dee4b010e --- /dev/null +++ b/crates/track/src/roto/maxflow.rs @@ -0,0 +1,438 @@ +//! Min-cut / max-flow on sparse graphs with source and sink terminals, by the +//! Boykov–Kolmogorov algorithm (Y. Boykov and V. Kolmogorov, "An Experimental Comparison of +//! Min-Cut/Max-Flow Algorithms for Energy Minimization in Vision", IEEE PAMI 26(9), 2004). +//! +//! Two search trees grow from the terminals (S from the source, T from the sink) through +//! non-saturated edges; when they touch, the path found is augmented; saturated tree edges turn +//! the nodes below them into *orphans*, which the adoption stage re-attaches to their tree (or +//! frees). Trees are reused between augmentations, which is what makes the method fast on the +//! grid graphs of image segmentation. The timestamp/distance heuristic of section 3.2.3 keeps +//! adoption paths short. +//! +//! Capacities are integers (callers scale their energies), so saturation tests are exact and +//! the result is deterministic. + +use std::collections::VecDeque; + +const NONE: u32 = u32::MAX; +const TERMINAL: u32 = u32::MAX - 1; +const ORPHAN: u32 = u32::MAX - 2; + +#[derive(Clone, Copy)] +struct Node { + first: u32, + /// Arc to the parent (from this node), or TERMINAL / ORPHAN / NONE (free). + parent: u32, + /// Residual terminal capacity: > 0 towards the source, < 0 towards the sink. + tr_cap: i64, + is_sink: bool, + active: bool, + ts: u32, + dist: u32, +} + +#[derive(Clone, Copy)] +struct Arc { + head: u32, + next: u32, + r_cap: i32, +} + +/// A graph for one min-cut computation. +pub struct Graph { + nodes: Vec, + arcs: Vec, + flow: i64, + time: u32, + queue: VecDeque, + orphans: VecDeque, +} + +/// Which side of the cut a node ended on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Segment { + Source, + Sink, +} + +impl Graph { + pub fn new(nodes: usize, edges_hint: usize) -> Graph { + Graph { + nodes: vec![Node { first: NONE, parent: NONE, tr_cap: 0, is_sink: false, active: false, ts: 0, dist: 0 }; nodes], + arcs: Vec::with_capacity(edges_hint * 2), + flow: 0, + time: 0, + queue: VecDeque::new(), + orphans: VecDeque::new(), + } + } + + pub fn len(&self) -> usize { + self.nodes.len() + } + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Terminal capacities of node `i`: `source` is paid when `i` ends on the sink side, `sink` + /// when it ends on the source side. + pub fn add_tweights(&mut self, i: usize, source: i64, sink: i64) { + let n = &mut self.nodes[i]; + let d = n.tr_cap; + let (s, t) = if d > 0 { (source + d, sink) } else { (source, sink - d) }; + self.flow += s.min(t); + n.tr_cap = s - t; + } + + /// An edge between `i` and `j` with capacity `cap` from i to j and `rev` from j to i. + pub fn add_edge(&mut self, i: usize, j: usize, cap: i32, rev: i32) { + debug_assert!(i != j); + let a = self.arcs.len() as u32; + self.arcs.push(Arc { head: j as u32, next: self.nodes[i].first, r_cap: cap }); + self.nodes[i].first = a; + self.arcs.push(Arc { head: i as u32, next: self.nodes[j].first, r_cap: rev }); + self.nodes[j].first = a + 1; + } + + #[inline] + fn set_active(&mut self, i: u32) { + let n = &mut self.nodes[i as usize]; + if !n.active { + n.active = true; + self.queue.push_back(i); + } + } + + /// Compute the maximum flow (= the minimum cut's cost). + pub fn maxflow(&mut self) -> i64 { + for i in 0..self.nodes.len() { + let n = &mut self.nodes[i]; + if n.tr_cap != 0 { + n.is_sink = n.tr_cap < 0; + n.parent = TERMINAL; + n.ts = 0; + n.dist = 1; + self.set_active(i as u32); + } + } + let mut current: Option = None; + loop { + let i = match current.take() { + Some(i) if self.nodes[i as usize].parent != NONE => i, + _ => loop { + let Some(i) = self.queue.pop_front() else { return self.flow }; + self.nodes[i as usize].active = false; + if self.nodes[i as usize].parent != NONE { + break i; + } + }, + }; + // Growth. + let mut found = NONE; + let ni = self.nodes[i as usize]; + let mut a = ni.first; + if !ni.is_sink { + while a != NONE { + let arc = self.arcs[a as usize]; + if arc.r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = false; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if nj.is_sink { + found = a; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } else { + while a != NONE { + let arc = self.arcs[a as usize]; + if self.arcs[(a ^ 1) as usize].r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = true; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if !nj.is_sink { + found = a ^ 1; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } + self.time = self.time.wrapping_add(1); + if found != NONE { + // Keep `i` current: it may have more paths. + current = Some(i); + self.augment(found); + self.adopt(); + } + } + } + + fn augment(&mut self, middle: u32) { + let mut b = self.arcs[middle as usize].r_cap as i64; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[(a ^ 1) as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(self.nodes[i as usize].tr_cap); + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[a as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(-self.nodes[i as usize].tr_cap); + let bi = b as i32; + self.arcs[(middle ^ 1) as usize].r_cap += bi; + self.arcs[middle as usize].r_cap -= bi; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[a as usize].r_cap += bi; + self.arcs[(a ^ 1) as usize].r_cap -= bi; + if self.arcs[(a ^ 1) as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap -= b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[(a ^ 1) as usize].r_cap += bi; + self.arcs[a as usize].r_cap -= bi; + if self.arcs[a as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap += b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + self.flow += b; + } + + fn make_orphan(&mut self, i: u32) { + self.nodes[i as usize].parent = ORPHAN; + self.orphans.push_back(i); + } + + /// Whether node `j`'s path to its terminal is intact; returns its distance (marking the + /// path with the current time) or None. + fn origin_dist(&mut self, j: u32) -> Option { + let mut d = 0u32; + let mut k = j; + loop { + let n = self.nodes[k as usize]; + if n.ts == self.time { + d += n.dist; + break; + } + let a = n.parent; + d += 1; + if a == TERMINAL { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = 1; + break; + } + if a == ORPHAN || a == NONE { + return None; + } + k = self.arcs[a as usize].head; + } + // Mark the path. + let total = d; + let mut k = j; + let mut dd = d; + while self.nodes[k as usize].ts != self.time { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = dd; + dd -= 1; + k = self.arcs[n.parent as usize].head; + } + Some(total) + } + + fn adopt(&mut self) { + while let Some(i) = self.orphans.pop_front() { + let sink = self.nodes[i as usize].is_sink; + let mut best = NONE; + let mut best_d = u32::MAX; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + // S orphan: the parent must push flow into i (residual on j→i = sister of i→j). + // T orphan: i must push flow into the parent (residual on i→j). + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink + && nj.parent != NONE + && let Some(d) = self.origin_dist(j) + && d < best_d + { + best = a; + best_d = d; + } + } + a = arc.next; + } + if best != NONE { + let n = &mut self.nodes[i as usize]; + n.parent = best; + n.ts = self.time; + n.dist = best_d + 1; + continue; + } + // No parent: free the node; its children become orphans, neighbours that could + // reach it become active. + self.nodes[i as usize].ts = 0; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink && nj.parent != NONE { + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + self.set_active(j); + } + let pa = nj.parent; + if pa != TERMINAL && pa != ORPHAN && self.arcs[pa as usize].head == i { + self.make_orphan(j); + } + } + a = arc.next; + } + self.nodes[i as usize].parent = NONE; + } + } + + /// The side of node `i` after [`Graph::maxflow`] (free nodes go to the sink). + pub fn segment(&self, i: usize) -> Segment { + let n = &self.nodes[i]; + if n.parent != NONE && !n.is_sink { Segment::Source } else { Segment::Sink } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Brute-force min cut over all labelings of a small graph. + fn brute(n: usize, t: &[(i64, i64)], e: &[(usize, usize, i32, i32)]) -> i64 { + let mut best = i64::MAX; + for m in 0..(1u32 << n) { + let src = |i: usize| m & (1 << i) != 0; + let mut c = 0i64; + for (i, (s, k)) in t.iter().enumerate() { + c += if src(i) { *k } else { *s }; + } + for (i, j, a, b) in e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + best = best.min(c); + } + best + } + + #[test] + fn matches_brute_force_on_random_graphs() { + let mut s = 12345u64; + let mut rnd = |m: u64| { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s % m + }; + for _ in 0..300 { + let n = 2 + rnd(9) as usize; + let t: Vec<(i64, i64)> = (0..n).map(|_| (rnd(20) as i64, rnd(20) as i64)).collect(); + let mut e = vec![]; + for _ in 0..rnd(20) { + let (i, j) = (rnd(n as u64) as usize, rnd(n as u64) as usize); + if i != j { + e.push((i, j, rnd(15) as i32, rnd(15) as i32)); + } + } + let mut g = Graph::new(n, e.len()); + for (i, (a, b)) in t.iter().enumerate() { + g.add_tweights(i, *a, *b); + } + for (i, j, a, b) in &e { + g.add_edge(*i, *j, *a, *b); + } + let f = g.maxflow(); + assert_eq!(f, brute(n, &t, &e)); + // The labeling realises the cut. + let src = |i: usize| g.segment(i) == Segment::Source; + let mut c = 0i64; + for (i, (a, b)) in t.iter().enumerate() { + c += if src(i) { *b } else { *a }; + } + for (i, j, a, b) in &e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + assert_eq!(c, f); + } + } +} diff --git a/crates/track/src/roto/mod.rs b/crates/track/src/roto/mod.rs new file mode 100644 index 000000000..393f743e0 --- /dev/null +++ b/crates/track/src/roto/mod.rs @@ -0,0 +1,754 @@ +//! Roto Brush segmentation and propagation, clean-room from published methods: +//! +//! - **Interactive graph-cut segmentation** (Y. Boykov and M.-P. Jolly, "Interactive Graph Cuts +//! for Optimal Boundary & Region Segmentation of Objects in N-D Images", ICCV 2001): the user's +//! foreground / background strokes are hard constraints, every other pixel gets a data cost +//! from colour models and neighbouring pixels a contrast-sensitive smoothness cost; the +//! minimum s–t cut ([`maxflow`], Boykov–Kolmogorov) is the segmentation. +//! - **Iterated colour models** (C. Rother, V. Kolmogorov and A. Blake, "GrabCut", SIGGRAPH +//! 2004): Gaussian mixtures ([`gmm`]) are fitted to the strokes, then re-fitted to the +//! segmentation and the cut repeated; the 8-connected n-link weight is +//! `γ/dist · exp(−β‖Iₚ − I_q‖²)` with `β = 1 / (2⟨‖Iₚ − I_q‖²⟩)`. +//! - **Coarse to fine**: large frames are segmented at a reduced size, then re-cut at full +//! resolution in a narrow band around the upsampled boundary (Lombaert et al., "A Multilevel +//! Banded Graph Cuts Method for Fast Image Segmentation", ICCV 2005). +//! - **Propagation** (in the spirit of X. Bai et al., "Video SnapCut", SIGGRAPH 2009): the +//! previous frame's matte is warped to the next frame by optical flow +//! (`effectcraft_raster::flow::block_flow`), and the frame re-segmented by graph cut only in a +//! band of **Search Radius** pixels around the warped boundary, with the previous frame's +//! colour models and the warped matte as a shape prior. Correction strokes on a frame re-cut +//! the whole frame with the warped matte as a soft prior, and propagation restarts from there. +//! +//! Mattes are in layer pixels at a given scale (1 = full resolution). Strokes are stored in +//! layer pixels ([`RotoData`], serde) and keyed per frame so that each frame's result is a +//! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes +//! on the way out to it ([`RotoData::chain_keys`]). + +pub mod gmm; +pub mod matting; +pub mod maxflow; +pub mod refine; +pub mod rle; + +use std::collections::BTreeMap; + +use effectcraft_raster::Image; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +pub use gmm::Gmm; +use maxflow::{Graph, Segment}; + +/// What a stroke paints. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum StrokeKind { + /// Roto Brush (green): foreground. + Fg, + /// Roto Brush with Alt (red): background. + Bg, + /// Refine Edge tool: mark a band for edge matting. + Refine, + /// Refine Edge tool with Alt: remove from the band. + RefineErase, +} + +impl StrokeKind { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "fg" | "foreground" => StrokeKind::Fg, + "bg" | "background" => StrokeKind::Bg, + "refine" | "refineedge" => StrokeKind::Refine, + "refineerase" | "refine-erase" | "erase" => StrokeKind::RefineErase, + _ => return None, + }) + } + pub fn is_segmentation(self) -> bool { + matches!(self, StrokeKind::Fg | StrokeKind::Bg) + } +} + +/// One stroke: a polyline swept by a disc of `radius` layer pixels, on layer frame `frame`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Stroke { + pub kind: StrokeKind, + pub frame: i64, + pub radius: f64, + pub points: Vec<[f64; 2]>, +} + +/// A Roto Brush instance's user data: strokes, the base frame and the segmentation span +/// (inclusive layer frames). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoData { + pub base: Option, + pub span: [i64; 2], + pub strokes: Vec, +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +impl RotoData { + pub fn from_json(s: &str) -> RotoData { + if s.trim().is_empty() { + return RotoData::default(); + } + serde_json::from_str(s).unwrap_or_default() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn is_empty(&self) -> bool { + self.base.is_none() || !self.strokes.iter().any(|s| s.kind.is_segmentation()) + } + pub fn strokes_at(&self, frame: i64) -> Vec<&Stroke> { + self.strokes.iter().filter(|s| s.frame == frame).collect() + } + /// Frames that carry strokes. + pub fn stroke_frames(&self) -> Vec { + let mut v: Vec = self.strokes.iter().map(|s| s.frame).collect(); + v.sort_unstable(); + v.dedup(); + v + } + pub fn in_span(&self, f: i64) -> bool { + self.base.is_some() && f >= self.span[0] && f <= self.span[1] + } + /// Add a stroke: the first segmentation stroke sets the base frame and a span of + /// `default_span` frames each side (clamped to `limits`); strokes outside the span extend it. + pub fn add(&mut self, s: Stroke, default_span: i64, limits: [i64; 2]) { + let f = s.frame; + match self.base { + None if s.kind.is_segmentation() => { + self.base = Some(f); + self.span = [(f - default_span).max(limits[0]), (f + default_span).min(limits[1]).max(f)]; + } + Some(_) => { + self.span[0] = self.span[0].min(f); + self.span[1] = self.span[1].max(f); + } + None => {} + } + self.strokes.push(s); + } + /// Largest Refine Edge stroke radius (layer pixels). + pub fn refine_radius(&self) -> f64 { + self.strokes.iter().filter(|s| matches!(s.kind, StrokeKind::Refine)).map(|s| s.radius).fold(0.0, f64::max) + } + /// Content key of every frame in the span: the base frame's from `seed` (what the frames + /// and settings are) and its strokes, every other frame's from its neighbour towards the + /// base and its own strokes. + pub fn chain_keys(&self, seed: u64) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(base) = self.base else { return out }; + let key = |prev: u64, f: i64| { + let mut h = prev ^ 0x9e37_79b9_7f4a_7c15; + fnv(&mut h, &f.to_le_bytes()); + for s in self.strokes.iter().filter(|s| s.frame == f) { + fnv(&mut h, serde_json::to_string(s).unwrap_or_default().as_bytes()); + } + h + }; + let k0 = key(seed, base); + out.insert(base, k0); + let mut k = k0; + for f in base + 1..=self.span[1] { + k = key(k, f); + out.insert(f, k); + } + k = k0; + for f in (self.span[0]..base).rev() { + k = key(k, f); + out.insert(f, k); + } + out + } + /// The neighbour a frame is propagated from (None for the base frame and outside the span). + pub fn source_of(&self, f: i64) -> Option { + let base = self.base?; + if !self.in_span(f) || f == base { + return None; + } + Some(if f > base { f - 1 } else { f + 1 }) + } +} + +/// Segmentation settings (the effect's Roto Brush parameters, in frame pixels). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SegOpts { + /// Pixels of the matte per layer pixel. + pub scale: f64, + /// Search Radius (layer pixels). + pub search_radius: f64, + /// Motion Threshold / Motion Damping, percent. + pub motion_threshold: f64, + pub motion_damping: f64, + /// Quality ▸ Best. + pub best: bool, +} + +impl Default for SegOpts { + fn default() -> Self { + SegOpts { scale: 1.0, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } + } +} + +/// One frame's segmentation: the binary matte, the Refine Edge band and the colour models. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FrameSeg { + pub w: usize, + pub h: usize, + pub matte: Vec, + pub refine: Vec, + pub fg: Gmm, + pub bg: Gmm, +} + +impl FrameSeg { + pub fn empty(w: usize, h: usize) -> FrameSeg { + FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], ..Default::default() } + } + pub fn area(&self) -> usize { + self.matte.iter().filter(|v| **v != 0).count() + } + /// Centroid of the matte (pixels), if any. + pub fn centroid(&self) -> Option<[f64; 2]> { + let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0); + for (i, v) in self.matte.iter().enumerate() { + if *v != 0 { + sx += (i % self.w) as f64 + 0.5; + sy += (i / self.w) as f64 + 0.5; + n += 1.0; + } + } + (n > 0.0).then(|| [sx / n, sy / n]) + } +} + +/// Straight RGB of an image (transparent pixels are black). +pub fn rgb_of(img: &Image) -> Vec<[f32; 3]> { + img.data.par_iter().map(|p| if p[3] > 1e-6 { [p[0] / p[3], p[1] / p[3], p[2] / p[3]] } else { [0.0; 3] }).collect() +} + +/// Pixels covered by `strokes` (of the given kinds) at `scale`. +pub fn rasterize(strokes: &[&Stroke], kinds: &[StrokeKind], scale: f64, w: usize, h: usize) -> Vec { + let mut m = vec![false; w * h]; + for s in strokes.iter().filter(|s| kinds.contains(&s.kind)) { + let r = (s.radius * scale).max(0.5); + let pts: Vec<[f64; 2]> = s.points.iter().map(|p| [p[0] * scale, p[1] * scale]).collect(); + let segs: Vec<([f64; 2], [f64; 2])> = if pts.len() == 1 { vec![(pts[0], pts[0])] } else { pts.windows(2).map(|w| (w[0], w[1])).collect() }; + for (a, b) in segs { + let x0 = (a[0].min(b[0]) - r).floor().max(0.0) as usize; + let x1 = ((a[0].max(b[0]) + r).ceil().max(0.0) as usize).min(w); + let y0 = (a[1].min(b[1]) - r).floor().max(0.0) as usize; + let y1 = ((a[1].max(b[1]) + r).ceil().max(0.0) as usize).min(h); + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + for y in y0..y1 { + for x in x0..x1 { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + let q = [a[0] + d[0] * t, a[1] + d[1] * t]; + if (p[0] - q[0]).hypot(p[1] - q[1]) <= r { + m[y * w + x] = true; + } + } + } + } + } + m +} + +/// Hard constraints: 1 = foreground, 2 = background, 0 = free. +const HARD_FG: u8 = 1; +const HARD_BG: u8 = 2; + +/// Graph-cut energy weights. +const GAMMA: f64 = 50.0; +const PRIOR: f64 = 1.0; +/// Fixed-point scale of the energies. +const K: f64 = 16.0; +const INF: i64 = 1 << 40; + +const NB: [(i64, i64, bool); 8] = [(1, 0, true), (0, 1, true), (1, 1, true), (-1, 1, true), (-1, 0, false), (0, -1, false), (-1, -1, false), (1, -1, false)]; + +fn beta(img: &[[f32; 3]], free: &[bool], w: usize, h: usize) -> f64 { + let (mut s, mut n) = (0.0f64, 0.0f64); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if !free[i] { + continue; + } + let c = img[i]; + for (dx, dy) in [(1usize, 0usize), (0, 1)] { + let (nx, ny) = (x + dx, y + dy); + if nx < w && ny < h { + let q = img[ny * w + nx]; + s += ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + n += 1.0; + } + } + } + } + if s <= 1e-12 { 1.0 } else { n / (2.0 * s) } +} + +/// One min-cut over the `free` pixels; others keep `labels`. `prior`: P(foreground) per pixel. +fn cut(img: &[[f32; 3]], w: usize, h: usize, labels: &mut [u8], free: &[bool], hard: &[u8], fg: &Gmm, bg: &Gmm, prior: Option<&[f32]>) { + let mut idx = vec![u32::MAX; w * h]; + let mut nodes = vec![]; + for i in 0..w * h { + if free[i] { + idx[i] = nodes.len() as u32; + nodes.push(i); + } + } + if nodes.is_empty() { + return; + } + let b = beta(img, free, w, h); + let costs: Vec<(f64, f64)> = nodes + .par_iter() + .map(|&i| { + let c = img[i]; + let (mut cf, mut cb) = (fg.cost(c), bg.cost(c)); + if let Some(p) = prior { + let pf = (p[i] as f64).clamp(0.02, 0.98); + cf += PRIOR * -pf.ln(); + cb += PRIOR * -(1.0 - pf).ln(); + } + (cf, cb) + }) + .collect(); + let mut g = Graph::new(nodes.len(), nodes.len() * 4); + for (n, &i) in nodes.iter().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let (cf, cb) = costs[n]; + let m = cf.min(cb); + let (mut src, mut snk) = (((cb - m) * K).round() as i64, ((cf - m) * K).round() as i64); + match hard[i] { + HARD_FG => (src, snk) = (INF, 0), + HARD_BG => (src, snk) = (0, INF), + _ => {} + } + g.add_tweights(n, src, snk); + let c = img[i]; + for (dx, dy, fwd) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + let q = img[j]; + let d2 = ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + let dist = if dx != 0 && dy != 0 { std::f64::consts::SQRT_2 } else { 1.0 }; + let wt = ((GAMMA / dist * (-b * d2).exp()) * K).round() as i32; + if wt <= 0 { + continue; + } + if idx[j] == u32::MAX { + // Fixed neighbour: disagreeing with it costs the edge. + if labels[j] != 0 { + g.add_tweights(n, wt as i64, 0); + } else { + g.add_tweights(n, 0, wt as i64); + } + } else if fwd { + g.add_edge(n, idx[j] as usize, wt, wt); + } + } + } + g.maxflow(); + for (n, &i) in nodes.iter().enumerate() { + labels[i] = (g.segment(n) == Segment::Source) as u8; + } +} + +/// Colour samples of the pixels where `sel` holds. +fn samples(img: &[[f32; 3]], sel: impl Fn(usize) -> bool) -> Vec<[f32; 3]> { + img.iter().enumerate().filter(|(i, _)| sel(*i)).map(|(_, c)| *c).collect() +} + +const COMPONENTS: usize = 5; +const MAX_SAMPLES: usize = 24_000; + +fn fit_models(img: &[[f32; 3]], labels: &[u8]) -> (Gmm, Gmm) { + let fg = samples(img, |i| labels[i] != 0); + let bg = samples(img, |i| labels[i] == 0); + (Gmm::fit(&fg, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bg, COMPONENTS, MAX_SAMPLES)) +} + +/// Keep the foreground components (8-connected) that touch `seeds`; fill small background +/// holes that touch neither the frame edge nor a background seed. +fn clean(labels: &mut [u8], seeds: &[bool], bg_seeds: &[bool], w: usize, h: usize) { + let n = w * h; + let mut comp = vec![u32::MAX; n]; + let mut stack = vec![]; + let mut fg_area = 0usize; + for start in 0..n { + if comp[start] != u32::MAX { + continue; + } + let lab = labels[start]; + let id = start as u32; + comp[start] = id; + stack.push(start); + let mut members = vec![]; + let (mut seeded, mut edge, mut bgs) = (false, false, false); + while let Some(i) = stack.pop() { + members.push(i); + seeded |= seeds[i]; + bgs |= bg_seeds[i]; + let (x, y) = ((i % w) as i64, (i / w) as i64); + edge |= x == 0 || y == 0 || x == w as i64 - 1 || y == h as i64 - 1; + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if comp[j] == u32::MAX && labels[j] == lab { + comp[j] = id; + stack.push(j); + } + } + } + if lab != 0 { + if seeded { + fg_area += members.len(); + } else { + for i in members { + labels[i] = 0; + } + } + } else if !edge && !bgs { + // Remember holes as negative candidates (resolved below once fg_area is known). + for i in members { + comp[i] = u32::MAX - 1; + } + } + } + // Fill holes smaller than 2 % of the foreground. + let mut seen = vec![false; n]; + for start in 0..n { + if comp[start] != u32::MAX - 1 || seen[start] || labels[start] != 0 { + continue; + } + let mut members = vec![start]; + seen[start] = true; + let mut k = 0; + while k < members.len() { + let i = members[k]; + k += 1; + let (x, y) = ((i % w) as i64, (i / w) as i64); + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if !seen[j] && comp[j] == u32::MAX - 1 && labels[j] == 0 { + seen[j] = true; + members.push(j); + } + } + } + // A hole is enclosed by foreground that survived the component filter. + let enclosed = members.iter().all(|&i| { + let (x, y) = ((i % w) as i64, (i / w) as i64); + NB.iter().all(|(dx, dy, _)| { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + return false; + } + let j = ny as usize * w + nx as usize; + labels[j] != 0 || comp[j] == u32::MAX - 1 + }) + }); + if enclosed && (members.len() as f64) < 0.02 * fg_area as f64 { + for i in members { + labels[i] = 1; + } + } + } +} + +fn downsample_rgb(img: &[[f32; 3]], w: usize, h: usize, f: usize) -> (Vec<[f32; 3]>, usize, usize) { + let (cw, ch) = (w.div_ceil(f), h.div_ceil(f)); + let out = (0..cw * ch) + .into_par_iter() + .map(|i| { + let (cx, cy) = (i % cw, i / cw); + let mut s = [0.0f32; 3]; + let mut n = 0.0; + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let c = img[y * w + x]; + s[0] += c[0]; + s[1] += c[1]; + s[2] += c[2]; + n += 1.0; + } + } + [s[0] / n, s[1] / n, s[2] / n] + }) + .collect(); + (out, cw, ch) +} + +/// What a frame is segmented with besides its strokes: the matte propagated from its neighbour. +pub struct Prior { + /// Warped neighbour matte, soft (0–1). + pub soft: Vec, + /// The neighbour's colour models. + pub fg: Gmm, + pub bg: Gmm, + /// Warped Refine Edge band. + pub refine: Vec, +} + +/// Segment a frame from its strokes (and a propagated prior): GrabCut iterations on a reduced +/// copy, then a full-resolution cut in a band around the boundary. +pub fn segment(img: &Image, strokes: &[&Stroke], prior: Option<&Prior>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (img.width as usize, img.height as usize); + if w == 0 || h == 0 { + return FrameSeg::empty(w, h); + } + let rgb = rgb_of(img); + let fgs = rasterize(strokes, &[StrokeKind::Fg], opts.scale, w, h); + let bgs = rasterize(strokes, &[StrokeKind::Bg], opts.scale, w, h); + let hard: Vec = fgs + .iter() + .zip(&bgs) + .map(|(f, b)| { + if *b { + HARD_BG + } else if *f { + HARD_FG + } else { + 0 + } + }) + .collect(); + let soft_prior: Option> = prior.map(|p| matting::gauss(&p.soft, w, h, (opts.search_radius * opts.scale / 4.0).max(1.0))); + // Initial colour models: the strokes (and the propagated matte). + let (fg0, bg0) = match prior { + Some(p) => { + let fgc = samples(&rgb, |i| (hard[i] == HARD_FG) || (hard[i] == 0 && p.soft[i] >= 0.5)); + let bgc = samples(&rgb, |i| (hard[i] == HARD_BG) || (hard[i] == 0 && p.soft[i] < 0.5)); + let f = Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES); + let b = Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES); + (if f.is_empty() { p.fg.clone() } else { f }, if b.is_empty() { p.bg.clone() } else { b }) + } + None => { + let fgc = samples(&rgb, |i| hard[i] == HARD_FG); + let mut bgc = samples(&rgb, |i| hard[i] == HARD_BG); + if bgc.is_empty() { + // No background strokes: what is far from the foreground strokes. + let d = matting::distance(&fgs, w, h); + let r = strokes.iter().filter(|s| s.kind == StrokeKind::Fg).map(|s| s.radius).fold(1.0, f64::max) * opts.scale; + let far = (4.0 * r).max(0.25 * w.max(h) as f64) as f32; + bgc = samples(&rgb, |i| d[i] > far); + if bgc.is_empty() { + bgc = samples(&rgb, |i| hard[i] != HARD_FG); + } + } + (Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES)) + } + }; + let iters = if opts.best { 4 } else { 3 }; + let work = if opts.best { 960 } else { 480 }; + let f = w.max(h).div_ceil(work).max(1); + let mut labels: Vec = match &soft_prior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => fgs.iter().map(|v| *v as u8).collect(), + }; + for (l, hd) in labels.iter_mut().zip(&hard) { + if *hd != 0 { + *l = (*hd == HARD_FG) as u8; + } + } + let (mut fgm, mut bgm) = (fg0, bg0); + if f > 1 { + let (crgb, cw, ch) = downsample_rgb(&rgb, w, h, f); + let mut chard = vec![0u8; cw * ch]; + let mut cprior = soft_prior.as_ref().map(|_| vec![0.0f32; cw * ch]); + for cy in 0..ch { + for cx in 0..cw { + let (mut anyf, mut anyb, mut s, mut n) = (false, false, 0.0f32, 0.0f32); + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let i = y * w + x; + anyf |= hard[i] == HARD_FG; + anyb |= hard[i] == HARD_BG; + if let Some(p) = &soft_prior { + s += p[i]; + } + n += 1.0; + } + } + chard[cy * cw + cx] = match (anyf, anyb) { + (true, false) => HARD_FG, + (false, true) => HARD_BG, + _ => 0, + }; + if let Some(cp) = &mut cprior { + cp[cy * cw + cx] = s / n; + } + } + } + let mut cl: Vec = match &cprior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => chard.iter().map(|v| (*v == HARD_FG) as u8).collect(), + }; + let free = vec![true; cw * ch]; + for it in 0..iters { + cut(&crgb, cw, ch, &mut cl, &free, &chard, &fgm, &bgm, cprior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&crgb, &cl); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + for y in 0..h { + for x in 0..w { + let i = y * w + x; + labels[i] = if hard[i] != 0 { (hard[i] == HARD_FG) as u8 } else { cl[(y / f) * cw + x / f] }; + } + } + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + // Full-resolution band. + let sd = matting::signed_distance(&labels, w, h); + let band = f as f32 + 1.5; + let free: Vec = sd.iter().map(|d| d.abs() <= band).collect(); + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + } else { + let free = vec![true; w * h]; + for it in 0..iters { + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + } + // Keep what the strokes (or the prior's core) select. + let mut seeds = fgs.clone(); + if let Some(p) = &soft_prior { + let core = core_of(p, w, h, opts.search_radius * opts.scale); + for (s, c) in seeds.iter_mut().zip(core) { + *s |= c; + } + } + clean(&mut labels, &seeds, &bgs, w, h); + finish(img, &rgb, labels, strokes, prior.map(|p| p.refine.as_slice()), opts, refine_radius) +} + +/// Pixels deep inside a soft matte (more than `r` from its edge), else the matte itself. +fn core_of(soft: &[f32], w: usize, h: usize, r: f64) -> Vec { + let bin: Vec = soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&bin, w, h); + let core: Vec = sd.iter().map(|d| *d as f64 > r).collect(); + if core.iter().any(|c| *c) { core } else { bin.iter().map(|v| *v != 0).collect() } +} + +/// Final models and Refine Edge band of a segmented frame. +fn finish(_img: &Image, rgb: &[[f32; 3]], labels: Vec, strokes: &[&Stroke], warped_refine: Option<&[u8]>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (_img.width as usize, _img.height as usize); + let (fg, bg) = fit_models(rgb, &labels); + let add = rasterize(strokes, &[StrokeKind::Refine], opts.scale, w, h); + let erase = rasterize(strokes, &[StrokeKind::RefineErase], opts.scale, w, h); + let near: Option> = warped_refine.filter(|r| r.iter().any(|v| *v != 0)).map(|_| matting::signed_distance(&labels, w, h)); + let rr = (refine_radius * opts.scale + 2.0) as f32; + let refine: Vec = (0..w * h) + .map(|i| { + let carried = match (warped_refine, &near) { + (Some(r), Some(sd)) => r[i] != 0 && sd[i].abs() <= rr, + _ => false, + }; + ((carried || add[i]) && !erase[i]) as u8 + }) + .collect(); + FrameSeg { w, h, matte: labels, refine, fg, bg } +} + +/// Warp the previous frame's segmentation onto the next frame by optical flow (next → prev), +/// with Motion Threshold / Damping applied to the vectors. +pub fn warp_prior(prev_img: &Image, prev: &FrameSeg, next_img: &Image, opts: &SegOpts) -> Prior { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let flow = effectcraft_raster::flow::block_flow(next_img, prev_img, 8, 4); + let thr = (opts.motion_threshold / 100.0 * opts.search_radius * opts.scale).max(0.0); + let damp = (1.0 - opts.motion_damping / 100.0).clamp(0.0, 1.0); + let src: Vec = prev.matte.iter().map(|v| *v as f32).collect(); + let pw = prev.w.min(w); + let ph = prev.h.min(h); + let res: Vec<(f32, u8)> = (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let v = flow.at(x, y); + let (mut vx, mut vy) = (v[0] as f64, v[1] as f64); + if vx.hypot(vy) < thr { + vx *= damp; + vy *= damp; + } + let (sx, sy) = (x + vx, y + vy); + let soft = if pw == prev.w && ph == prev.h { matting::sample(&src, prev.w, prev.h, sx, sy) } else { 0.0 }; + let (ix, iy) = (sx.floor().clamp(0.0, (prev.w.max(1) - 1) as f64) as usize, sy.floor().clamp(0.0, (prev.h.max(1) - 1) as f64) as usize); + let r = prev.refine.get(iy * prev.w + ix).copied().unwrap_or(0); + (soft, r) + }) + .collect(); + Prior { soft: res.iter().map(|r| r.0).collect(), fg: prev.fg.clone(), bg: prev.bg.clone(), refine: res.iter().map(|r| r.1).collect() } +} + +/// Propagate the previous frame's segmentation to the next frame: warp it, then re-cut in a +/// band of Search Radius pixels around the warped boundary. Strokes on the next frame re-cut +/// the whole frame with the warped matte as a soft prior instead. +pub fn propagate(prev_img: &Image, prev: &FrameSeg, next_img: &Image, strokes: &[&Stroke], opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let prior = warp_prior(prev_img, prev, next_img, opts); + if strokes.iter().any(|s| s.kind.is_segmentation()) { + return segment(next_img, strokes, Some(&prior), opts, refine_radius); + } + let rgb = rgb_of(next_img); + let mut labels: Vec = prior.soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&labels, w, h); + let r = (opts.search_radius * opts.scale).max(2.0) as f32; + let free: Vec = sd.iter().map(|d| d.abs() <= r).collect(); + let hard = vec![0u8; w * h]; + let soft = matting::gauss(&prior.soft, w, h, (r as f64 / 4.0).max(1.0)); + let (fg, bg) = if prev.fg.is_empty() || prev.bg.is_empty() { fit_models(&rgb, &labels) } else { (prev.fg.clone(), prev.bg.clone()) }; + cut(&rgb, w, h, &mut labels, &free, &hard, &fg, &bg, Some(&soft)); + let seeds = core_of(&prior.soft, w, h, r as f64); + clean(&mut labels, &seeds, &vec![false; w * h], w, h); + finish(next_img, &rgb, labels, strokes, Some(&prior.refine), opts, refine_radius) +} + +/// Intersection over union of two binary mattes. +pub fn iou(a: &[u8], b: &[u8]) -> f64 { + let (mut i, mut u) = (0usize, 0usize); + for (x, y) in a.iter().zip(b) { + let (x, y) = (*x != 0, *y != 0); + i += (x && y) as usize; + u += (x || y) as usize; + } + if u == 0 { 1.0 } else { i as f64 / u as f64 } +} + +#[cfg(test)] +mod tests; diff --git a/crates/track/src/roto/refine.rs b/crates/track/src/roto/refine.rs new file mode 100644 index 000000000..2a498e286 --- /dev/null +++ b/crates/track/src/roto/refine.rs @@ -0,0 +1,198 @@ +//! From a frame's binary segmentation to the final soft matte: the Roto Brush Matte and Refine +//! Edge Matte adjustments, edge matting inside the Refine Edge band (colour guided filter), +//! Reduce Chatter (temporal averaging with the neighbouring frames), matte motion blur and edge +//! colour decontamination. + +use super::FrameSeg; +use super::matting::{self, contrast, gauss, guided_filter, shift_soft, signed_distance}; + +/// The effect's matte parameters (After Effects' units: pixels and percent). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MatteParams { + // Roto Brush Matte. + pub rb_feather: f64, + pub rb_contrast: f64, + pub rb_shift: f64, + pub rb_chatter: f64, + // Refine Edge Matte. + pub smooth: f64, + pub feather: f64, + pub contrast: f64, + pub shift: f64, + pub chatter: f64, + pub motion_blur: bool, + pub mb_samples: usize, + pub shutter_angle: f64, + /// Higher Quality: closed-form matting after the guided-filter estimate. + pub higher_quality: bool, + pub decontaminate: bool, + pub decontamination: f64, + pub extend_where_smoothed: bool, + pub increase_radius: f64, +} + +impl Default for MatteParams { + fn default() -> Self { + MatteParams { + rb_feather: 0.0, + rb_contrast: 0.0, + rb_shift: 0.0, + rb_chatter: 0.0, + smooth: 1.0, + feather: 0.0, + contrast: 0.0, + shift: 0.0, + chatter: 0.0, + motion_blur: false, + mb_samples: 11, + shutter_angle: 180.0, + higher_quality: true, + decontaminate: true, + decontamination: 100.0, + extend_where_smoothed: true, + increase_radius: 0.0, + } + } +} + +fn shifted(seg: &FrameSeg, px: f64) -> Vec { + if px.abs() < 0.25 { + return seg.matte.iter().map(|v| *v as f32).collect(); + } + let sd = signed_distance(&seg.matte, seg.w, seg.h); + sd.iter().map(|d| ((*d as f64) > -px) as u8 as f32).collect() +} + +/// The soft matte of `seg` (all planes `seg.w × seg.h`), with `rgb` the frame's straight colour. +/// `prev` / `next`: the neighbouring frames' segmentations (Reduce Chatter, motion blur). +/// `band_radius`: the Refine Edge brush radius in these pixels; `scale`: pixels per layer pixel. +pub fn matte_alpha( + rgb: &[[f32; 3]], + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + p: &MatteParams, + band_radius: f64, + scale: f64, +) -> Vec { + let (w, h) = (seg.w, seg.h); + let n = w * h; + let same = |s: &&FrameSeg| s.w == w && s.h == h; + let (prev, next) = (prev.filter(same), next.filter(same)); + // Roto Brush Matte: Shift Edge (100 % = 10 px), Reduce Chatter, Feather, Contrast. + let shift_px = p.rb_shift / 100.0 * 10.0 * scale; + let mut a = shifted(seg, shift_px); + let nb: Vec> = [prev, next].into_iter().flatten().map(|s| shifted(s, shift_px)).collect(); + if p.rb_chatter > 0.0 && !nb.is_empty() { + let k = (p.rb_chatter / 100.0).clamp(0.0, 1.0) as f32 * (nb.len() as f32 / (nb.len() as f32 + 1.0)); + for i in 0..n { + let m = nb.iter().map(|v| v[i]).sum::() / nb.len() as f32; + a[i] += (m - a[i]) * k; + } + } + if p.rb_feather > 0.0 { + a = gauss(&a, w, h, p.rb_feather * scale / 2.0); + } + contrast(&mut a, p.rb_contrast); + // Refine Edge band: guided-filter matting. + if seg.refine.iter().any(|v| *v != 0) && seg.refine.len() == n { + let r = (band_radius * 1.0).round().max(2.0) as usize; + let eps = (2e-4 * (1.0 + p.smooth.max(0.0))) as f32; + // Work on the band's bounding box (plus the filter's reach). + let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0, 0); + for (i, v) in seg.refine.iter().enumerate() { + if *v != 0 { + let (x, y) = (i % w, i / w); + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x + 1); + y1 = y1.max(y + 1); + } + } + let pad = 2 * r + 2; + let (x0, y0, x1, y1) = (x0.saturating_sub(pad), y0.saturating_sub(pad), (x1 + pad).min(w), (y1 + pad).min(h)); + let (cw, ch) = (x1 - x0, y1 - y0); + let crop = |src: &dyn Fn(usize) -> f32| -> Vec { (0..cw * ch).map(|j| src((y0 + j / cw) * w + x0 + j % cw)).collect() }; + let cimg: Vec<[f32; 3]> = (0..cw * ch).map(|j| rgb[(y0 + j / cw) * w + x0 + j % cw]).collect(); + let cp = crop(&|i| a[i]); + let band: Vec = crop(&|i| seg.refine[i] as f32).iter().map(|v| *v > 0.0).collect(); + let outside: Vec = band.iter().map(|b| !b).collect(); + // Guided filter estimate, then closed-form matting with everything outside the band known. + let mut q = guided_filter(&cimg, &cp, cw, ch, r, eps); + for j in 0..cw * ch { + q[j] = if band[j] { q[j].clamp(0.0, 1.0) } else { cp[j] }; + } + if p.higher_quality { + q = matting::closed_form(&cimg, &q, &outside, cw, ch, 1, 1e-5 * (1.0 + p.smooth.max(0.0)) as f32, 50); + } + // Refine Edge Matte adjustments. + if p.smooth > 1.0 { + q = gauss(&q, cw, ch, (p.smooth - 1.0) * 0.5 * scale); + } + if p.feather > 0.0 { + q = gauss(&q, cw, ch, p.feather * scale / 2.0); + } + contrast(&mut q, p.contrast); + shift_soft(&mut q, p.shift); + // Blend in, fading over the band's last two pixels. + let din = matting::distance(&outside, cw, ch); + for j in 0..cw * ch { + if !band[j] { + continue; + } + let wgt = (din[j] / 2.0).clamp(0.0, 1.0); + let i = (y0 + j / cw) * w + x0 + j % cw; + a[i] += (q[j] - a[i]) * wgt; + } + // Refine-stage Reduce Chatter: towards the neighbours' mattes inside the band. + if p.chatter > 0.0 && !nb.is_empty() { + let k = (p.chatter / 100.0).clamp(0.0, 1.0) as f32 * 0.5; + let soft: Vec> = nb.iter().map(|v| gauss(v, w, h, (r as f64 / 2.0).max(1.0))).collect(); + for i in 0..n { + if seg.refine[i] != 0 { + let m = soft.iter().map(|v| v[i]).sum::() / soft.len() as f32; + a[i] += (m - a[i]) * k; + } + } + } + } + // Motion blur: the matte's motion between the neighbouring frames. + if p.motion_blur { + let c = seg.centroid(); + let v = match (prev.and_then(FrameSeg::centroid), next.and_then(FrameSeg::centroid), c) { + (Some(a), Some(b), _) => Some([(b[0] - a[0]) / 2.0, (b[1] - a[1]) / 2.0]), + (Some(a), None, Some(c)) => Some([c[0] - a[0], c[1] - a[1]]), + (None, Some(b), Some(c)) => Some([b[0] - c[0], b[1] - c[1]]), + _ => None, + }; + if let Some(v) = v { + let s = p.shutter_angle / 360.0; + a = matting::motion_blur(&a, w, h, [v[0] * s, v[1] * s], p.mb_samples); + } + } + a.iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0)); + a +} + +/// Decontaminate edge colours of `rgb` in place for the matte `alpha` (see +/// [`matting::decontaminate`]); returns the decontamination map. `band`: the Refine Edge band +/// (pixels where decontamination always applies). +pub fn decontaminate(rgb: &mut [[f32; 3]], alpha: &[f32], band: &[u8], w: usize, h: usize, p: &MatteParams, band_radius: f64, scale: f64) -> Vec { + if !p.decontaminate { + return vec![0.0; w * h]; + } + let grow = p.increase_radius * scale; + let in_band: Vec = if band.len() == w * h && band.iter().any(|v| *v != 0) { + if grow > 0.5 { + let d = matting::distance(&band.iter().map(|v| *v != 0).collect::>(), w, h); + d.iter().map(|v| (*v as f64) <= grow).collect() + } else { + band.iter().map(|v| *v != 0).collect() + } + } else { + vec![false; w * h] + }; + let region: Vec = (0..w * h).map(|i| alpha[i] > 0.004 && alpha[i] < 0.996 && (in_band[i] || p.extend_where_smoothed)).collect(); + let radius = (band_radius + grow).max(4.0 * scale).round() as usize; + matting::decontaminate(rgb, alpha, ®ion, w, h, radius, (p.decontamination / 100.0).clamp(0.0, 1.0) as f32) +} diff --git a/crates/track/src/roto/rle.rs b/crates/track/src/roto/rle.rs new file mode 100644 index 000000000..308b2689a --- /dev/null +++ b/crates/track/src/roto/rle.rs @@ -0,0 +1,136 @@ +//! Compact text encodings of mattes for project storage: run-length coding (LEB128 varints) +//! wrapped in standard base64 (RFC 4648 §4). + +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +pub fn base64_encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let b = [c[0], *c.get(1).unwrap_or(&0), *c.get(2).unwrap_or(&0)]; + let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32; + out.push(B64[(n >> 18) as usize & 63] as char); + out.push(B64[(n >> 12) as usize & 63] as char); + out.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' }); + } + out +} + +pub fn base64_decode(s: &str) -> Option> { + let val = |c: u8| -> Option { + Some(match c { + b'A'..=b'Z' => (c - b'A') as u32, + b'a'..=b'z' => (c - b'a' + 26) as u32, + b'0'..=b'9' => (c - b'0' + 52) as u32, + b'+' => 62, + b'/' => 63, + _ => return None, + }) + }; + let bytes: Vec = s.bytes().filter(|c| !c.is_ascii_whitespace()).collect(); + if !bytes.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(bytes.len() / 4 * 3); + for c in bytes.chunks(4) { + let pad = c.iter().rev().take_while(|b| **b == b'=').count(); + let mut n = 0u32; + for (i, b) in c.iter().enumerate() { + n |= if *b == b'=' { 0 } else { val(*b)? } << (18 - 6 * i); + } + out.push((n >> 16) as u8); + if pad < 2 { + out.push((n >> 8) as u8); + } + if pad < 1 { + out.push(n as u8); + } + } + Some(out) +} + +fn put(out: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +fn get(data: &[u8], pos: &mut usize) -> Option { + let mut v = 0u64; + let mut shift = 0; + loop { + let b = *data.get(*pos)?; + *pos += 1; + v |= ((b & 0x7f) as u64) << shift; + if b & 0x80 == 0 { + return Some(v); + } + shift += 7; + if shift > 63 { + return None; + } + } +} + +/// Encode a byte plane as (value, run length) pairs. +pub fn encode(values: &[u8]) -> String { + let mut out = vec![]; + put(&mut out, values.len() as u64); + let mut i = 0; + while i < values.len() { + let v = values[i]; + let mut j = i + 1; + while j < values.len() && values[j] == v { + j += 1; + } + out.push(v); + put(&mut out, (j - i) as u64); + i = j; + } + base64_encode(&out) +} + +/// Decode [`encode`]'s output (None when malformed). +pub fn decode(s: &str) -> Option> { + let data = base64_decode(s)?; + let mut pos = 0; + let n = get(&data, &mut pos)? as usize; + if n > 1 << 30 { + return None; + } + let mut out = Vec::with_capacity(n); + while out.len() < n { + let v = *data.get(pos)?; + pos += 1; + let run = get(&data, &mut pos)? as usize; + if out.len() + run > n { + return None; + } + out.resize(out.len() + run, v); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips() { + for n in [0usize, 1, 2, 3, 4, 5, 100, 1000] { + let v: Vec = (0..n).map(|i| ((i / 7) % 3) as u8 * 100).collect(); + assert_eq!(decode(&encode(&v)).unwrap(), v); + let raw: Vec = (0..n).map(|i| (i * 31) as u8).collect(); + assert_eq!(base64_decode(&base64_encode(&raw)).unwrap(), raw); + } + assert_eq!(base64_encode(b"Man"), "TWFu"); + assert_eq!(base64_encode(b"Ma"), "TWE="); + assert!(decode("!!!!").is_none()); + } +} diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rs new file mode 100644 index 000000000..bec3f3783 --- /dev/null +++ b/crates/track/src/roto/tests.rs @@ -0,0 +1,331 @@ +//! Roto Brush on synthetic sequences: a textured disk moving over a textured background. + +use super::refine::{MatteParams, decontaminate, matte_alpha}; +use super::*; + +pub(crate) const W: u32 = 200; +pub(crate) const H: u32 = 150; +pub(crate) const R: f64 = 32.0; + +/// Smooth value noise in 0..1. +pub(crate) fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +pub(crate) fn fg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 11); + [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32] +} + +pub(crate) fn bg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 5); + let m = noise(x * 1.7, y * 1.7, 9); + [(0.05 + 0.25 * m) as f32, (0.3 + 0.4 * n) as f32, (0.45 + 0.4 * m) as f32] +} + +pub(crate) fn center(f: i64) -> [f64; 2] { + let t = f as f64; + [60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()] +} + +/// Frame `f`: the disk (texture moving with it) over the static background, with a 1-pixel +/// anti-aliased edge; `c` overrides the centre. +pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0u8; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let f = fg_color(p[0] - c[0], p[1] - c[1]); + let b = bg_color(p[0], p[1]); + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = (d <= R) as u8; + } + } + (img, gt) +} + +fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke { + Stroke { kind, frame, radius, points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect() } +} + +/// The strokes an artist would paint on frame `f`: one through the disk, one around it. +pub(crate) fn base_strokes(f: i64) -> Vec { + let c = center(f); + vec![ + line(StrokeKind::Fg, f, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0), + line(StrokeKind::Bg, f, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0), + line(StrokeKind::Bg, f, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0), + ] +} + +#[test] +fn strokes_segment_the_base_frame() { + let (img, gt) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); + // Deterministic. + let again = segment(&img, &refs, None, &SegOpts::default(), 0.0); + assert_eq!(seg, again); +} + +#[test] +fn coarse_to_fine_segments_large_frames() { + // A 2× upscaled frame goes through the reduced-size path (Standard works at 480 px). + let (small, _) = frame_at(center(0)); + let img = effectcraft_raster::resample(&small, W * 3, H * 3); + let c = center(0); + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut s| { + s.points.iter_mut().for_each(|p| *p = [p[0] * 3.0, p[1] * 3.0]); + s.radius *= 3.0; + s + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let gt: Vec = + (0..W * H * 9).map(|i| ((((i % (W * 3)) as f64 + 0.5) / 3.0 - c[0]).hypot((((i / (W * 3)) as f64 + 0.5) / 3.0) - c[1]) <= R) as u8).collect(); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); +} + +#[test] +fn propagation_follows_the_disk_for_20_frames() { + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts::default(); + let mut seg = segment(&img0, &refs, None, &opts, 0.0); + let mut prev = img0; + let mut worst: f64 = 1.0; + for f in 1..=20 { + let (img, gt) = frame_at(center(f)); + seg = propagate(&prev, &seg, &img, &[], &opts, 0.0); + let j = iou(&seg.matte, >); + worst = worst.min(j); + prev = img; + } + assert!(worst > 0.9, "worst IoU {worst}"); +} + +#[test] +fn a_correction_stroke_fixes_a_bad_frame() { + let opts = SegOpts::default(); + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg0 = segment(&img0, &refs, None, &opts, 0.0); + // The disk jumps far away on frame 1: propagation cannot follow. + let jump = [150.0, 90.0]; + let (img1, gt1) = frame_at(jump); + let bad = propagate(&img0, &seg0, &img1, &[], &opts, 0.0); + assert!(iou(&bad.matte, >1) < 0.5, "the jump should break propagation"); + // Paint the disk in and the old place out. + let c0 = center(0); + let fix = [ + line(StrokeKind::Fg, 1, [jump[0] - 15.0, jump[1]], [jump[0] + 15.0, jump[1]], 5.0), + line(StrokeKind::Bg, 1, [c0[0] - 10.0, c0[1] - 10.0], [c0[0] + 10.0, c0[1] + 10.0], 6.0), + ]; + let fr: Vec<&Stroke> = fix.iter().collect(); + let good = propagate(&img0, &seg0, &img1, &fr, &opts, 0.0); + let j = iou(&good.matte, >1); + assert!(j > 0.95, "corrected IoU {j}"); + // And propagation continues from the corrected frame. + let c2 = [jump[0] + 3.0, jump[1] + 1.0]; + let (img2, gt2) = frame_at(c2); + let next = propagate(&img1, &good, &img2, &[], &opts, 0.0); + assert!(iou(&next.matte, >2) > 0.93); +} + +/// A soft-edged disk (alpha ramp over `width` pixels) over a background. +fn soft_frame(c: [f64; 2], width: f64, fg: [f32; 3], bg: [f32; 3]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0.0f32; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = ((R + width / 2.0 - d) / width).clamp(0.0, 1.0) as f32; + // Faint texture on both layers. + let tf = (noise(p[0], p[1], 3) as f32 - 0.5) * 0.04; + let tb = (noise(p[0], p[1], 4) as f32 - 0.5) * 0.04; + let f = [fg[0] + tf, fg[1] + tf, fg[2] + tf]; + let b = [bg[0] + tb, bg[1] + tb, bg[2] + tb]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = a; + } + } + (img, gt) +} + +fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke { + Stroke { + kind: StrokeKind::Refine, + frame, + radius, + points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(), + } +} + +#[test] +fn refine_edge_recovers_a_soft_edge_and_decontaminates() { + let c = [100.0, 75.0]; + let (fgc, bgc) = ([0.9f32, 0.25, 0.1], [0.1f32, 0.3, 0.9]); + let (img, gt) = soft_frame(c, 10.0, fgc, bgc); + let mut st = vec![ + line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0), + line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0), + ]; + st.push(ring(0, c, R, 9.0)); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 9.0); + assert!(seg.refine.iter().filter(|v| **v != 0).count() > 1000); + let p = MatteParams::default(); + let rgb = rgb_of(&img); + let a = matte_alpha(&rgb, &seg, None, None, &p, 9.0, 1.0); + // Inside the ramp the matte is fractional and close to the truth. + let (mut err, mut n, mut frac) = (0.0f64, 0usize, 0usize); + let (mut err_bin, mut n_bin) = (0.0f64, 0usize); + for i in 0..a.len() { + if gt[i] > 0.0 && gt[i] < 1.0 { + err += (a[i] - gt[i]).abs() as f64; + err_bin += (seg.matte[i] as f32 - gt[i]).abs() as f64; + n += 1; + n_bin += 1; + frac += (a[i] > 0.05 && a[i] < 0.95) as usize; + } + } + let (err, err_bin) = (err / n as f64, err_bin / n_bin as f64); + assert!(err < 0.03, "mean alpha error {err} (binary {err_bin})"); + assert!(err < err_bin * 0.4); + assert!(frac as f64 > 0.6 * n as f64, "{frac}/{n} fractional"); + // Decontamination: edge pixels' colour is the foreground's, not a blue blend. + let mut col = rgb.clone(); + let map = decontaminate(&mut col, &a, &seg.refine, W as usize, H as usize, &p, 9.0, 1.0); + let (mut blue_before, mut blue_after, mut m) = (0.0f64, 0.0f64, 0usize); + for i in 0..a.len() { + if a[i] > 0.2 && a[i] < 0.8 { + blue_before += rgb[i][2] as f64; + blue_after += col[i][2] as f64; + m += 1; + } + } + let (bb, ba) = (blue_before / m as f64, blue_after / m as f64); + assert!(ba < 0.2 && bb > 0.35, "blue before {bb}, after {ba}"); + assert!(map.iter().any(|v| *v > 0.2)); + // Without the band, the binary matte passes through. + let mut seg2 = seg.clone(); + seg2.refine.iter_mut().for_each(|v| *v = 0); + let a2 = matte_alpha(&rgb, &seg2, None, None, &p, 9.0, 1.0); + assert!(a2.iter().zip(&seg.matte).all(|(x, y)| (*x - *y as f32).abs() < 1e-6)); +} + +#[test] +fn reduce_chatter_and_motion_blur_use_the_neighbours() { + let (w, h) = (40usize, 20usize); + let mk = |x0: usize| FrameSeg { + w, + h, + matte: (0..w * h).map(|i| (i % w >= x0 && i % w < x0 + 10) as u8).collect(), + refine: vec![0; w * h], + ..Default::default() + }; + let (a, b, c) = (mk(10), mk(14), mk(18)); + let rgb = vec![[0.5f32; 3]; w * h]; + let p = MatteParams { rb_chatter: 100.0, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + // Pixel 12 is only in the previous frame: partially on. + assert!(al[12] > 0.2 && al[12] < 0.5, "{}", al[12]); + let p = MatteParams { motion_blur: true, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + assert!(al[14] > 0.3 && al[14] < 0.9, "{}", al[14]); + assert!(al[19] > 0.99); +} + +#[test] +fn chain_keys_restart_at_correction_frames() { + let mut d = RotoData::default(); + for s in base_strokes(5) { + d.add(s, 20, [0, 30]); + } + assert_eq!(d.base, Some(5)); + assert_eq!(d.span, [0, 25]); + let k1 = d.chain_keys(1); + d.add(line(StrokeKind::Fg, 12, [0.0, 0.0], [1.0, 1.0], 3.0), 20, [0, 30]); + let k2 = d.chain_keys(1); + for f in 0..=11 { + assert_eq!(k1[&f], k2[&f], "frame {f}"); + } + for f in 12..=25 { + assert_ne!(k1[&f], k2[&f], "frame {f}"); + } + assert_ne!(d.chain_keys(2)[&5], k2[&5]); + let back = RotoData::from_json(&d.to_json()); + assert_eq!(back, d); + assert_eq!(d.source_of(12), Some(11)); + assert_eq!(d.source_of(3), Some(4)); + assert_eq!(d.source_of(5), None); +} + +/// `cargo test -p effectcraft-track --release roto_perf -- --ignored --nocapture` +#[test] +#[ignore] +fn roto_perf_1080p() { + let up = |c: [f64; 2]| { + let (img, _) = frame_at(c); + effectcraft_raster::resample(&img, 1920, 1080) + }; + let s = 1920.0 / W as f64; + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut k| { + k.points.iter_mut().for_each(|p| *p = [p[0] * s, p[1] * s]); + k.radius *= s; + k + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts { search_radius: 15.0 * s / 4.0, ..Default::default() }; + let a = up(center(0)); + let t = std::time::Instant::now(); + let seg = segment(&a, &refs, None, &opts, 0.0); + eprintln!("segment 1080p: {:?}", t.elapsed()); + let b = up(center(1)); + let t = std::time::Instant::now(); + let next = propagate(&a, &seg, &b, &[], &opts, 0.0); + eprintln!("propagate 1080p: {:?} (area {})", t.elapsed(), next.area()); + let mut seg2 = next.clone(); + let c = center(1); + let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect(); + let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring }]; + let r = rasterize(&st.iter().collect::>(), &[StrokeKind::Refine], 1.0, 1920, 1080); + seg2.refine = r.iter().map(|v| *v as u8).collect(); + let rgb = rgb_of(&b); + let t = std::time::Instant::now(); + let _ = matte_alpha(&rgb, &seg2, None, None, &MatteParams::default(), 20.0, 1.0); + eprintln!("refine 1080p: {:?}", t.elapsed()); +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 3644d7ac0..91d7c335c 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -34,11 +34,12 @@ pub enum PanelKind { History, Markers, MaskInterpolation, + ScriptConsole, EssentialGraphics, } impl PanelKind { - pub const ALL: [PanelKind; 25] = [ + pub const ALL: [PanelKind; 26] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -63,6 +64,7 @@ impl PanelKind { PanelKind::History, PanelKind::Markers, PanelKind::MaskInterpolation, + PanelKind::ScriptConsole, PanelKind::EssentialGraphics, ]; pub fn title(self) -> &'static str { @@ -91,6 +93,7 @@ impl PanelKind { PanelKind::History => "History", PanelKind::Markers => "Markers", PanelKind::MaskInterpolation => "Mask Interpolation", + PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", } } @@ -175,7 +178,43 @@ fn vsplit(size: SplitSize, a: DockNode, b: DockNode) -> DockNode { DockNode::Split { vertical: true, size, a: Box::new(a), b: Box::new(b) } } -pub const WORKSPACES: [&str; 10] = ["Default", "Standard", "Small Screen", "Animation", "Effects", "Motion Tracking", "Paint", "Text", "Minimal", "All Panels"]; +/// The built-in workspaces: After Effects 2026's (Window ▸ Workspace), less Libraries (an Adobe +/// service). Learn shows the Home screen (community links instead of Adobe's tutorials); +/// Essential Graphics uses the Properties panel; Undocked Panels floats the panels over the +/// Composition panel. +pub const WORKSPACES: &[&str] = &[ + "Default", + "Standard", + "Small Screen", + "Animation", + "Effects", + "Motion Tracking", + "Paint", + "Text", + "Minimal", + "All Panels", + "Review", + "Learn", + "Essential Graphics", + "Color", + "Undocked Panels", +]; + +/// The floating panels a workspace starts with (Undocked Panels), in points for a window of +/// about 1680×1020 (they are kept inside the window when drawn). +pub fn workspace_floating(name: &str) -> Vec { + use PanelKind::*; + let f = |panels: &[PanelKind], rect: [f32; 4]| Floating { panels: panels.to_vec(), active: 0, rect }; + match name { + "Undocked Panels" => vec![ + f(&[Project, EffectControls], [24.0, 110.0, 320.0, 420.0]), + f(&[Preview, Info, Audio], [1330.0, 110.0, 320.0, 300.0]), + f(&[EffectsPresets, Properties, Character, Paragraph], [1330.0, 430.0, 320.0, 360.0]), + f(&[Timeline, RenderQueue], [24.0, 640.0, 1280.0, 340.0]), + ], + _ => vec![], + } +} /// The default layout of a named workspace. pub fn workspace(name: &str) -> DockNode { @@ -229,6 +268,40 @@ pub fn workspace(name: &str) -> DockNode { tabs(&[Timeline], 0), ), "Minimal" => vsplit(Ratio(0.6), tabs(&[Composition], 0), tabs(&[Timeline], 0)), + // Review: a large viewer with the markers and the project beside it. + "Review" => vsplit( + Ratio(0.66), + hsplit(FixedA(280.0), tabs(&[Project, Markers], 0), hsplit(FixedB(260.0), tabs(&[Composition, Layer], 0), right(&[Preview, Info], &[Audio]))), + tabs(&[Timeline, RenderQueue], 0), + ), + // Essential Graphics: the Properties panel (After Effects' successor to the Essential + // Graphics panel) full height on the right, type panels beside the viewer. + "Essential Graphics" => hsplit( + FixedB(320.0), + vsplit( + Ratio(0.58), + hsplit( + FixedA(280.0), + tabs(&[Project, EffectControls], 0), + hsplit(FixedB(260.0), tabs(&[Composition, Layer], 0), right(&[Character, Paragraph], &[Align, Preview])), + ), + tabs(&[Timeline], 0), + ), + stack(&[(Properties, true, None), (EffectsPresets, false, None)]), + ), + // Color: wide Effect Controls for grading effects, Info for colour readouts. + "Color" => vsplit( + Ratio(0.6), + hsplit( + FixedA(360.0), + tabs(&[EffectControls, Project], 0), + hsplit(FixedB(300.0), tabs(&[Composition, Layer], 0), right(&[Info, Preview], &[EffectsPresets])), + ), + tabs(&[Timeline], 0), + ), + // Undocked Panels: the Composition panel docked, the others floating + // ([`workspace_floating`]). + "Undocked Panels" => tabs(&[Composition, Layer, Flowchart], 0), "Small Screen" => vsplit( Ratio(0.55), hsplit( @@ -817,8 +890,59 @@ mod tests { fn workspaces_contain_core_panels() { for w in WORKSPACES { let d = workspace(w); - assert!(d.contains(PanelKind::Timeline), "{w}"); + let floating: Vec = workspace_floating(w).into_iter().flat_map(|f| f.panels).collect(); + assert!(d.contains(PanelKind::Timeline) || floating.contains(&PanelKind::Timeline), "{w}"); assert!(d.contains(PanelKind::Composition) || d.contains(PanelKind::Layer), "{w}"); + // A panel is in one place only. + let mut all = vec![]; + d.panels(&mut all); + all.extend(floating); + let n = all.len(); + all.sort_by_key(|p| p.id()); + all.dedup(); + assert_eq!(all.len(), n, "{w} has a panel twice"); + } + } + + #[test] + fn after_effects_workspaces_exist() { + // Window ▸ Workspace in After Effects 2026, less Libraries (an Adobe service). + for w in [ + "Default", + "Review", + "Learn", + "Small Screen", + "Standard", + "All Panels", + "Animation", + "Essential Graphics", + "Color", + "Effects", + "Minimal", + "Motion Tracking", + "Paint", + "Text", + "Undocked Panels", + ] { + assert!(WORKSPACES.contains(&w), "{w}"); + } + assert!(!WORKSPACES.contains(&"Libraries")); + // Layouts differ from Default (except Learn, which is Default plus the Home screen). + let def = workspace("Default"); + for w in WORKSPACES.iter().filter(|w| !matches!(**w, "Default" | "Learn")) { + assert_ne!(workspace(w), def, "{w} falls back to Default"); + } + assert!(workspace("Essential Graphics").contains(PanelKind::Properties)); + assert!(workspace("Color").contains(PanelKind::EffectControls)); + assert!(!workspace_floating("Undocked Panels").is_empty()); + // Every built-in workspace is in Window ▸ Workspace. + let menu: Vec = effectcraft_engine::menus::entries() + .into_iter() + .filter(|(_, e)| e.command == "window.workspace") + .filter_map(|(_, e)| e.params.get("name").and_then(|v| v.as_str()).map(str::to_string)) + .collect(); + for w in WORKSPACES { + assert!(menu.iter().any(|m| m == w), "{w} missing from Window > Workspace"); } } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 62239f191..d0e5a6d93 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -7,6 +7,11 @@ //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. //! +//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also +//! written to disk under a content key (the comp's content hash, frame, scale, view and render +//! options), and a job first looks there: frames survive restarts and project reopenings. The +//! timeline shows disk-only frames as a blue cache bar. +//! //! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set //! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer //! registers with egui-wgpu and draws directly; pixels are read back only when something needs @@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, Gpu}; @@ -122,6 +128,37 @@ pub struct RenderSource { /// Show Channel / exposure / region-of-interest drawing read the CPU frame texture, so GPU /// frames are read back (the compositing still runs on the GPU). pub gpu_display: bool, + /// The persistent disk cache, when enabled. + pub disk: Option>, +} + +/// Content keys of (project revision, comp), computed once per revision. +type ContentKeys = Arc>>; + +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { + return k; + } + let k = disk_cache::comp_content_key(project, ItemId(comp)); + if let Ok(mut m) = keys.lock() { + if m.len() > 256 { + m.clear(); + } + m.insert((revision, comp), k); + } + k +} + +fn opts_hash(o: &RenderOpts) -> u64 { + let mut h = disk_cache::Hash128::default(); + h.write(format!("{o:?}").as_bytes()); + h.finish() as u64 +} + +/// Disk key of a viewer frame (render options included: region of interest, draft…). +fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { + let content = content_key(keys, project, key.revision, key.comp); + disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } pub struct Frames { @@ -135,6 +172,7 @@ pub struct Frames { ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, + content_keys: ContentKeys, } impl Default for Frames { @@ -149,6 +187,7 @@ impl Default for Frames { pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"), ctx: None, last_ms: Arc::new(Mutex::new(0.0)), + content_keys: Arc::default(), } } } @@ -195,6 +234,21 @@ impl Frames { v } + /// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar. + /// `opts` are the viewer's render options (part of the disk key). + pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec { + let Some(dc) = &src.disk else { return vec![] }; + let ram: HashSet = self.cached_frames(revision, comp, scale).into_iter().collect(); + let oh = opts_hash(opts); + (0..frames.clamp(0, 100_000)) + .filter(|f| !ram.contains(f)) + .filter(|f| { + let key = FrameKey { revision, comp, frame: *f, scale, view }; + dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh)) + }) + .collect() + } + /// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest /// frames when it shrinks. pub fn set_budget(&self, bytes: usize) { @@ -278,7 +332,14 @@ impl Frames { } fn worker(&self) -> Worker { - Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() } + Worker { + queue: self.queue.clone(), + cache: self.cache.clone(), + inflight: self.inflight.clone(), + ctx: self.ctx.clone(), + last: self.last_ms.clone(), + content_keys: self.content_keys.clone(), + } } /// Render queued frames on this thread, most important first, until the queue is empty or @@ -306,12 +367,13 @@ struct Worker { inflight: Arc>>, ctx: Option, last: Arc>, + content_keys: ContentKeys, } impl Worker { /// Render the queued job that matters most right now. `false` when the queue was empty. fn run_next(&self) -> bool { - let Worker { queue, cache, inflight, ctx, last } = self; + let Worker { queue, cache, inflight, ctx, last, content_keys } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; // Urgent first (newest), then prefetch in request order. @@ -325,19 +387,18 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); - r.expr = job.src.expr.as_deref(); - r.cache = Some(&job.src.layer_cache); - r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); - // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no - // adapter, or a frame the GPU cannot finish here) the CPU renders it. - let gpu_frame = match (&job.src.gpu, r.active_accel()) { - (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), - _ => None, - }; - let ci = match gpu_frame { - Some(f) => FrameImage::Gpu(Arc::new(f)), - None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); + let ci = match from_disk { + Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))), + None => { + let ci = self.render_job(&job); + if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba); + } + ci + } }; if job.urgent && let Ok(mut l) = last.lock() @@ -361,6 +422,23 @@ impl Worker { } true } + + fn render_job(&self, job: &Job) -> FrameImage { + let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); + r.expr = job.src.expr.as_deref(); + r.cache = Some(&job.src.layer_cache); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), + _ => None, + }; + match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + } + } } struct Job { diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index d28cb9500..d2de8fc38 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -33,6 +33,7 @@ pub enum Icon { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, // switches & timeline Eye, @@ -282,6 +283,15 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.ellipse(6.0, 10.0, 3.6, 3.6); pen.dot(6.0, 10.0, 1.0); } + RefineEdge => { + // A brush over a soft (dashed) edge. + pen.line(&[(13.5, 2.5), (9.0, 7.0)]); + pen.ellipse(7.0, 9.0, 2.6, 2.6); + pen.line(&[(2.0, 14.0), (3.5, 12.5)]); + pen.line(&[(5.0, 14.0), (6.5, 12.5)]); + pen.line(&[(8.0, 14.0), (9.5, 12.5)]); + pen.line(&[(11.0, 14.0), (12.5, 12.5)]); + } Puppet => { pen.circle(8.0, 3.6, 1.6); pen.line(&[(8.0, 5.2), (8.0, 10.0)]); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index cc12a3416..286073796 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -14,6 +14,7 @@ pub mod frames; pub mod header; pub mod icons; pub mod menus; +pub mod native_menu; pub mod panels; pub mod state; pub mod theme; @@ -86,6 +87,11 @@ pub struct Hooks { pub pick_folder: Option Option>>, /// Save dialog for other file kinds: (default name, extension). pub pick_save_file: Option Option>>, + /// The system clipboard's text (native menu Edit ▸ Paste into a text field). + pub clipboard_text: Option Option>>, + /// Application actions the OS performs (`app.hide`, `app.hideOthers`, `app.showAll` on + /// macOS). Returns false when the host doesn't handle the id. + pub app_action: Option bool>>, } #[derive(Default)] @@ -158,6 +164,10 @@ pub struct EffectcraftApp { /// Docked groups laid out last frame: (active panel, group rect) — `~` maximizes the one /// under the pointer. pub(crate) dock_rects: Vec<(PanelKind, egui::Rect)>, + /// Home screen: recent-project thumbnail textures by path (None = no thumbnail). + pub(crate) home_thumbs: std::collections::HashMap>, + /// The (project path, saved revision) whose thumbnail was stored last. + pub(crate) home_thumb_saved: Option<(String, u64)>, } impl EffectcraftApp { @@ -202,6 +212,8 @@ impl EffectcraftApp { applied_prefs: None, recovery: None, dock_rects: vec![], + home_thumbs: Default::default(), + home_thumb_saved: None, } .with_ui_commands() } @@ -267,8 +279,12 @@ impl EffectcraftApp { pub fn set_workspace(&mut self, name: &str) { self.ui.workspace = name.to_string(); self.ui.dock = self.ui.saved_workspaces.get(name).cloned().unwrap_or_else(|| dock::workspace(name)); - self.ui.floating = self.ui.saved_floating.get(name).cloned().unwrap_or_default(); + self.ui.floating = self.ui.saved_floating.get(name).cloned().unwrap_or_else(|| dock::workspace_floating(name)); self.ui.maximized = None; + // Learn: the Home screen (community links) in the Composition panel. + if name == "Learn" { + self.ui.start_screen = true; + } } /// Built-in workspaces followed by the saved ones (Window ▸ Workspace ▸ Save as New Workspace). @@ -308,6 +324,7 @@ impl EffectcraftApp { layer_cache: self.session.layer_cache.clone(), gpu: self.gpu.clone(), gpu_display: false, + disk: self.session.disk_cache.clone(), } } @@ -405,13 +422,11 @@ impl EffectcraftApp { self.request_frame_with(comp, frame, scale, true); } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { - let Some(c) = self.session.project.comp(comp) else { return }; - let key = self.frame_key(comp, frame, scale); - let t = c.frame_rate.tick_of(frame); + /// Render options of viewer frames of `comp` at `scale`. + pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); - let opts = RenderOpts { + RenderOpts { scale, motion_blur: true, guides: true, @@ -420,7 +435,14 @@ impl EffectcraftApp { roi, backend: effectcraft_engine::render::Backend::Auto, proxy: Default::default(), - }; + } + } + + fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + let Some(c) = self.session.project.comp(comp) else { return }; + let key = self.frame_key(comp, frame, scale); + let t = c.frame_rate.tick_of(frame); + let opts = self.frame_opts(comp, scale); if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { @@ -726,6 +748,12 @@ impl EffectcraftApp { // Warp Stabilizer: finish analyses and start queued (re-)analyses in the background. if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() { self.session.poll_warp(true); + self.session.poll_roto(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } self.apply_prefs(&ctx); @@ -757,6 +785,7 @@ impl EffectcraftApp { let body = egui::Rect::from_min_max(egui::pos2(full.min.x + 4.0, header.max.y + 2.0), egui::pos2(full.max.x - 4.0, full.max.y - 4.0)); self.dock_area(ui, body); panels::precomp::mini_flowchart(self, &ctx); + panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index f97851f88..689963f71 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -59,6 +59,21 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("timeline.reveal.feather", "Reveal Mask Feather", [], Some("F")), uic!("timeline.reveal.levels", "Reveal Audio Levels (press twice: Waveform)", [], Some("L")), uic!("timeline.reveal.waveform", "Reveal Audio Waveform", [], None), + uic!("timeline.revealAdd.position", "Add Position to Revealed Properties", [], Some("Shift+P")), + uic!("timeline.revealAdd.scale", "Add Scale to Revealed Properties", [], Some("Shift+S")), + uic!("timeline.revealAdd.rotation", "Add Rotation to Revealed Properties", [], Some("Shift+R")), + uic!("timeline.revealAdd.opacity", "Add Opacity to Revealed Properties", [], Some("Shift+T")), + uic!("timeline.revealAdd.anchor", "Add Anchor Point to Revealed Properties", [], Some("Shift+A")), + uic!("timeline.revealAdd.effects", "Add Effects to Revealed Properties", [], Some("Shift+E")), + uic!("timeline.revealAdd.masks", "Add Masks to Revealed Properties", [], Some("Shift+M")), + uic!("timeline.revealAdd.feather", "Add Mask Feather to Revealed Properties", [], Some("Shift+F")), + uic!("timeline.revealAdd.levels", "Add Audio Levels to Revealed Properties", [], Some("Shift+L")), + uic!("timeline.revealAdd.animated", "Add Animated Properties to Revealed Properties", [], Some("Shift+U")), + uic!("timeline.column", "Show/Hide Timeline Column", [], None), + uic!("timeline.sourceName", "Source Name / Layer Name", [], None), + uic!("timeline.search", "Search Timeline", [], None), + uic!("flowchart.options", "Flowchart Options", [], None), + uic!("flowchart.graph", "Flowchart Graph", [], None), uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")), uic!("tool.selection", "Selection Tool", [], Some("V")), uic!("tool.hand", "Hand Tool", [], Some("H")), @@ -72,7 +87,7 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("tool.pen", "Pen Tool", [], Some("G")), uic!("tool.type", "Type Tools", [], Some("Cmd+T")), uic!("tool.brush", "Brush Tools", [], Some("Cmd+B")), - uic!("tool.rotoBrush", "Roto Brush Tool", [], Some("Alt+W")), + uic!("tool.rotoBrush", "Roto Brush & Refine Edge Tools", [], Some("Alt+W")), uic!("tool.puppet", "Puppet Tools", [], Some("Cmd+P")), uic!("app.newComp", "New Composition...", [], None), uic!("app.compSettings", "Composition Settings...", [], None), @@ -89,26 +104,42 @@ pub fn panel_command_id(p: PanelKind) -> String { format!("window.panel.{}", p.id()) } -fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64) { +/// The reveal shortcuts (P, S, R, T, A, E, M, F, L…). `add` (Shift+key) adds the property to +/// (or removes it from) those already revealed instead of replacing them. +pub fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64, add: bool) { // AE's double-press shortcuts: L then L again quickly reveals the Waveform (LL). let kind = match app.last_reveal.take() { Some((k, t)) if k == "levels" && kind == "levels" && now - t < 0.6 => "waveform", _ => kind, }; app.last_reveal = Some((kind.to_string(), now)); - let add = app.ui.timeline.reveal.contains(&kind.to_string()); - if add && app.ui.timeline.reveal.len() == 1 { - app.ui.timeline.reveal.clear(); - app.ui.timeline.open_layers.clear(); + let tl = &mut app.ui.timeline; + let present = tl.reveal.iter().any(|k| k == kind); + let sel: Vec = if app.session.state.selected_layers.is_empty() { + app.session.active_comp().map(|c| c.layers.iter().map(|l| l.id.0).collect()).unwrap_or_default() + } else { + app.session.state.selected_layers.iter().map(|l| l.0).collect() + }; + if add { + if present { + tl.reveal.retain(|k| k != kind); + if tl.reveal.is_empty() { + tl.open_layers.clear(); + } + } else { + tl.reveal.retain(|k| k != "props" || !tl.reveal_props.is_empty()); + tl.reveal.push(kind.to_string()); + tl.open_layers.extend(sel); + } return; } - app.ui.timeline.reveal = vec![kind.to_string()]; - let sel = if app.session.state.selected_layers.is_empty() { - app.session.active_comp().map(|c| c.layers.iter().map(|l| l.id).collect()).unwrap_or_default() - } else { - app.session.state.selected_layers.clone() - }; - app.ui.timeline.open_layers = sel.iter().map(|l| l.0).collect(); + if present && tl.reveal.len() == 1 { + tl.reveal.clear(); + tl.open_layers.clear(); + return; + } + tl.reveal = vec![kind.to_string()]; + tl.open_layers = sel.into_iter().collect(); } fn no_params(p: &Value) -> bool { @@ -139,6 +170,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V "pen" => Some(9), "type" => Some(10), "brush" => Some(11), + "rotoBrush" => Some(14), "puppet" => Some(15), _ => None, }; @@ -166,9 +198,30 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V if k == "animated" { return run_engine(app, ctx, "anim.reveal", json!({"kind": "keyframes"})); } - reveal(app, k, now); + reveal(app, k, now, false); return Ok(Value::Null); } + if let Some(k) = id.strip_prefix("timeline.revealAdd.") { + reveal(app, k, now, true); + return Ok(json!({"revealed": app.ui.timeline.reveal})); + } + // U reveals keyframed properties; UU (pressed twice quickly) all modified properties; U + // again later hides them. + if id == "anim.reveal" && params.get("kind").and_then(Value::as_str) == Some("keyframes") { + let last = app.last_reveal.take(); + app.last_reveal = Some(("keyframes".into(), now)); + match last { + Some((k, t)) if k == "keyframes" && now - t < 0.6 => { + return run_engine(app, ctx, "anim.reveal", json!({"kind": "modified"})); + } + Some((k, _)) if k == "keyframes" && app.ui.timeline.reveal == ["props"] => { + app.ui.timeline.reveal.clear(); + app.ui.timeline.open_layers.clear(); + return Ok(json!({"revealed": 0})); + } + _ => {} + } + } // Docking: {panel, anchor, zone: center|left|right|top|bottom}, {panel, rect?}, {panel?}. if let Some(op) = id.strip_prefix("window.").filter(|o| matches!(*o, "maximizePanel" | "dockPanel" | "floatPanel" | "closePanel")) { let panel_of = |k: &str| params.get(k).and_then(Value::as_str).map(|s| PanelKind::from_name(s).ok_or(format!("unknown panel `{s}`"))); @@ -268,6 +321,24 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V return Ok(Value::Null); } "timeline.zoomFit" => app.ui.timeline.pps = None, + "timeline.column" => { + let col = params.get("column").and_then(Value::as_str).ok_or("timeline.column: need `column`")?; + let on = params.get("visible").and_then(Value::as_bool).unwrap_or(!crate::panels::timeline::column_visible(&app.ui.timeline, col)); + crate::panels::timeline::set_column(&mut app.ui.timeline, col, on)?; + let shown: Vec<&str> = + crate::panels::timeline::COLUMNS.iter().map(|c| c.0).filter(|c| crate::panels::timeline::column_visible(&app.ui.timeline, c)).collect(); + return Ok(json!({"columns": shown})); + } + "timeline.sourceName" => { + let tl = &mut app.ui.timeline; + tl.source_name = params.get("value").and_then(Value::as_bool).unwrap_or(!tl.source_name); + return Ok(json!({"sourceName": tl.source_name})); + } + "flowchart.options" | "flowchart.graph" => return crate::panels::flowchart::command(app, id, ¶ms), + "timeline.search" => { + app.ui.timeline.search = params.get("query").and_then(Value::as_str).unwrap_or_default().to_string(); + app.show_panel(PanelKind::Timeline); + } "timeline.graphEditor" => app.ui.timeline.graph_editor = !app.ui.timeline.graph_editor, "timeline.switchesModes" => app.ui.timeline.show_modes = !app.ui.timeline.show_modes, "timeline.collapseAll" => { @@ -399,8 +470,16 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu ); Value::Null } - "app.hide" => { - ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)); + "app.hide" | "app.hideOthers" | "app.showAll" => { + // The desktop app asks macOS (NSApplication hide / hideOtherApplications / + // unhideAllApplications); elsewhere Hide minimizes and the others don't apply. + if app.hooks.app_action.as_ref().is_some_and(|f| f(id)) { + return Ok(Value::Null); + } + match id { + "app.hide" => ctx.send_viewport_cmd(egui::ViewportCommand::Minimized(true)), + _ => return Err(format!("`{id}` is only available on macOS")), + } Value::Null } "app.quit" => { @@ -637,6 +716,13 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu Value::Null } "comp.flowchart" => { + // Chart the active comp; Layer ▸ Reveal ▸ Reveal Layer in Project Flowchart shows + // layers and selects the layer's node. + app.ui.flowchart.root = app.session.active_comp_id().map(|c| c.0); + if let (Some(c), Some(l)) = (app.session.active_comp_id(), app.session.state.selected_layers.first()) { + app.ui.flowchart.layers = true; + app.ui.flowchart.selected = Some(format!("layer:{}:{}", c.0, l.0)); + } app.show_panel(PanelKind::Flowchart); Value::Null } @@ -697,7 +783,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "render.saveCurrentPreview" => ("path", Ask::Save("Preview.mp4")), - "file.runScript" => ("path", Ask::Open(&["jsonl", "json", "txt"])), + "file.runScript" => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), @@ -719,7 +805,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, @@ -742,6 +828,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option Vec<&'static str> { effectcraft_engine::menus::top_level() } -fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { +pub(crate) fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { match e.command.as_str() { "edit.undo" => app.session.history.undo.last().map(|u| format!("Undo {}", u.0)).unwrap_or_else(|| "Can't Undo".into()), "edit.redo" => app.session.history.redo.last().map(|u| format!("Redo {}", u.0)).unwrap_or_else(|| "Can't Redo".into()), @@ -780,12 +870,12 @@ fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { } /// The entry's shortcut in the active keyboard shortcut preset. -fn entry_shortcut(app: &EffectcraftApp, e: &MenuEntry) -> Option { +pub(crate) fn entry_shortcut(app: &EffectcraftApp, e: &MenuEntry) -> Option { app.session.shortcuts().shortcut_of(&e.command, &e.params).map(str::to_string) } /// Check-mark state of frontend toggles (engine state is answered by the engine). -fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { +pub(crate) fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { let v = &app.ui.viewer; let pstr = |k: &str| e.params.get(k).and_then(Value::as_str); match e.command.as_str() { @@ -817,7 +907,7 @@ fn entry_checked(app: &EffectcraftApp, e: &MenuEntry) -> Option { } } -fn entry_enabled(app: &EffectcraftApp, e: &MenuEntry) -> bool { +pub(crate) fn entry_enabled(app: &EffectcraftApp, e: &MenuEntry) -> bool { if e.command == "window.panel" { return true; } diff --git a/crates/ui-egui/src/native_menu.rs b/crates/ui-egui/src/native_menu.rs new file mode 100644 index 000000000..ef5070b36 --- /dev/null +++ b/crates/ui-egui/src/native_menu.rs @@ -0,0 +1,459 @@ +//! The native (macOS) menu bar as a plain, serializable spec. The desktop app turns it into an +//! `NSMenu` with `muda` (`apps/effectcraft/src/native_menu.rs`); everything that decides *what* +//! the menu contains lives here, so it is tested without creating real menus and builds for the +//! web (where the in-window menu bar is used). +//! +//! The spec mirrors the engine's After Effects menu tree ([`effectcraft_engine::menus`]): +//! submenus, separators, labels (Undo/Redo and label names are live), check marks, enabled +//! state, and the active shortcut preset's keys as accelerators. File ▸ Open Recent lists the +//! recent projects. The application menu's Hide / Hide Others / Show All are macOS' +//! predefined items, and a Services submenu is added as on every Mac app. +//! +//! Item ids are positional (`m.3.0.2`), stable while the menu's structure stays the same; the +//! app rebuilds the native menu when [`NativeMenu::structure`] changes and otherwise only pushes +//! label / enabled / checked updates, and only when [`state_key`] changes. + +use std::hash::{Hash, Hasher}; + +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use effectcraft_engine::menus::{MenuNode, menu_bar}; + +/// What macOS performs itself (no command dispatch). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Role { + Services, + Hide, + HideOthers, + ShowAll, +} + +/// One node of the native menu. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum NativeNode { + Item(NativeItem), + Separator, + Submenu { + label: String, + children: Vec, + }, + /// A predefined macOS item; `command` is the engine entry it stands for (empty for Services). + Predefined { + role: Role, + label: String, + command: String, + }, +} + +/// A clickable item: activating it runs `command` with `params`, exactly like the in-window bar. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct NativeItem { + pub id: String, + pub label: String, + pub command: String, + #[serde(skip_serializing_if = "Value::is_null")] + pub params: Value, + /// The shortcut in EffectCraft notation (`Cmd+Shift+S`). + pub shortcut: Option, + /// The menu key equivalent in `muda` accelerator notation, when the shortcut can be one (see + /// [`accelerator`]). + pub accelerator: Option, + pub enabled: bool, + pub checked: Option, +} + +/// The whole menu bar: top-level menus in order (the application menu first). +#[derive(Clone, Debug, Default, PartialEq, Serialize)] +pub struct NativeMenu { + pub menus: Vec, +} + +impl NativeMenu { + /// Every clickable item, depth first. + pub fn items(&self) -> Vec<&NativeItem> { + fn rec<'a>(n: &'a [NativeNode], out: &mut Vec<&'a NativeItem>) { + for x in n { + match x { + NativeNode::Item(i) => out.push(i), + NativeNode::Submenu { children, .. } => rec(children, out), + _ => {} + } + } + } + let mut out = vec![]; + rec(&self.menus, &mut out); + out + } + + pub fn find(&self, id: &str) -> Option<&NativeItem> { + self.items().into_iter().find(|i| i.id == id) + } + + /// Fingerprint of everything that needs a rebuild when it changes (tree shape, commands, + /// accelerators, submenu labels); labels, enabled and check state are updated in place. + pub fn structure(&self) -> u64 { + fn rec(n: &[NativeNode], h: &mut std::collections::hash_map::DefaultHasher) { + for x in n { + match x { + NativeNode::Item(i) => { + 0u8.hash(h); + i.id.hash(h); + i.command.hash(h); + i.params.to_string().hash(h); + i.accelerator.hash(h); + } + NativeNode::Separator => 1u8.hash(h), + NativeNode::Submenu { label, children } => { + 2u8.hash(h); + label.hash(h); + rec(children, h); + 3u8.hash(h); + } + NativeNode::Predefined { role, .. } => { + 4u8.hash(h); + role.hash(h); + } + } + } + } + let mut h = std::collections::hash_map::DefaultHasher::new(); + rec(&self.menus, &mut h); + h.finish() + } +} + +/// Build the native menu spec for the app's current state. +pub fn build(app: &EffectcraftApp) -> NativeMenu { + fn rec(app: &EffectcraftApp, nodes: &[MenuNode], path: &str, out: &mut Vec) { + let mut services_added = false; + for (i, n) in nodes.iter().enumerate() { + let id = format!("{path}.{i}"); + match n { + MenuNode::Separator => out.push(NativeNode::Separator), + MenuNode::Submenu { label, children } => { + let mut kids = vec![]; + if label == "Open Recent" { + for (k, p) in app.session.prefs.recent_projects.iter().enumerate() { + let name = std::path::Path::new(p).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(p.clone()); + kids.push(NativeNode::Item(NativeItem { + id: format!("{id}.r{k}"), + label: name, + command: "file.openRecent".into(), + params: json!({"index": k}), + shortcut: None, + accelerator: None, + enabled: true, + checked: None, + })); + } + if !kids.is_empty() { + kids.push(NativeNode::Separator); + } + } + rec(app, children, &id, &mut kids); + out.push(NativeNode::Submenu { label: label.clone(), children: kids }); + } + MenuNode::Item(e) => { + let role = match e.command.as_str() { + "app.hide" => Some(Role::Hide), + "app.hideOthers" => Some(Role::HideOthers), + "app.showAll" => Some(Role::ShowAll), + _ => None, + }; + if let Some(role) = role { + // Services sits just above Hide, as in every Mac app. + if role == Role::Hide && !services_added { + out.push(NativeNode::Predefined { role: Role::Services, label: "Services".into(), command: String::new() }); + out.push(NativeNode::Separator); + services_added = true; + } + out.push(NativeNode::Predefined { role, label: e.label.clone(), command: e.command.clone() }); + continue; + } + let shortcut = crate::menus::entry_shortcut(app, e); + out.push(NativeNode::Item(NativeItem { + id, + label: crate::menus::entry_label(app, e), + command: e.command.clone(), + params: e.params.clone(), + accelerator: shortcut.as_deref().and_then(accelerator), + shortcut, + enabled: crate::menus::entry_enabled(app, e), + checked: crate::menus::entry_checked(app, e), + })); + } + } + } + } + let mut menus = vec![]; + rec(app, menu_bar(), "m", &mut menus); + NativeMenu { menus } +} + +/// The `muda` accelerator for an EffectCraft shortcut, or `None` when it must stay a plain +/// keyboard shortcut. Only shortcuts with Cmd or Ctrl (and function keys) become menu key +/// equivalents: a single-key shortcut (`V`, `Space`, `U`) or an Alt/Shift-only one would be +/// swallowed by the menu while typing into a text field. Those still work through the +/// window's shortcut dispatcher, as on the other platforms. +pub fn accelerator(shortcut: &str) -> Option { + let parts: Vec<&str> = match shortcut.strip_suffix("++") { + Some(head) => head.split('+').chain(std::iter::once("+")).collect(), + None => shortcut.split('+').collect(), + }; + let (key, mods) = parts.split_last()?; + let mut out: Vec = vec![]; + let mut primary = false; + for m in mods { + out.push( + match *m { + "Cmd" => { + primary = true; + "Cmd" + } + "Ctrl" => { + primary = true; + "Ctrl" + } + "Alt" => "Alt", + "Shift" => "Shift", + _ => return None, + } + .to_string(), + ); + } + let function_key = key.len() >= 2 && key.starts_with('F') && key[1..].parse::().is_ok_and(|n| (1..=24).contains(&n)); + if !primary && !function_key { + return None; + } + let k = match *key { + "Num*" => "NumpadMultiply".to_string(), + "+" => "Equal".to_string(), + k if k.starts_with("Num") && k.len() == 4 => format!("Numpad{}", &k[3..]), + "Escape" | "Space" | "Enter" | "Tab" | "Home" | "End" | "PageUp" | "PageDown" | "Delete" | "Backspace" => key.to_string(), + "ArrowLeft" | "ArrowRight" | "ArrowUp" | "ArrowDown" => key.to_string(), + k if k.chars().count() == 1 => { + let c = k.chars().next()?; + if c.is_ascii_alphanumeric() || "`\\[],-=./;'".contains(c) { + c.to_ascii_uppercase().to_string() + } else { + return None; + } + } + k if function_key => k.to_string(), + _ => return None, + }; + out.push(k); + Some(out.join("+")) +} + +/// Cheap per-frame fingerprint of everything menu labels, check marks and enabled state depend +/// on (project revision, selection, history, settings, executed commands, viewer toggles). +pub fn state_key(app: &EffectcraftApp) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + let s = &app.session; + s.revision.hash(&mut h); + s.prefs_revision.hash(&mut h); + s.path.hash(&mut h); + s.active_comp_id().map(|c| c.0).hash(&mut h); + for l in &s.state.selected_layers { + l.0.hash(&mut h); + } + s.state.selected_keys.len().hash(&mut h); + s.state.selected_props.len().hash(&mut h); + s.state.project_selection.len().hash(&mut h); + s.state.selected_vertices.len().hash(&mut h); + s.state.clipboard.len().hash(&mut h); + s.state.key_clipboard.len().hash(&mut h); + (s.history.undo.len(), s.history.redo.len()).hash(&mut h); + s.journal.len().hash(&mut h); + s.journal.last().map(|j| &j.0).hash(&mut h); + s.prefs.recent_projects.hash(&mut h); + s.keymaps.active.hash(&mut h); + let v = &app.ui.viewer; + (v.rulers, v.guides, v.snap_guides, v.lock_guides, v.grid, v.snap_grid, v.show_layer_controls, v.pasteboard).hash(&mut h); + v.res.label().hash(&mut h); + (app.ui.cache_when_idle, app.ui.preview_audio, &app.ui.workspace, app.dialog.is_some()).hash(&mut h); + h.finish() +} + +/// Commands that act on a focused text field instead (Edit menu while typing): the egui event to +/// send in their place. +fn text_field_event(command: &str, clipboard: Option) -> Option> { + let key = |k: egui::Key, shift: bool| egui::Event::Key { + key: k, + physical_key: None, + pressed: true, + repeat: false, + modifiers: egui::Modifiers { shift, ..egui::Modifiers::COMMAND }, + }; + Some(match command { + "edit.copy" => vec![egui::Event::Copy], + "edit.cut" => vec![egui::Event::Cut], + "edit.paste" => vec![egui::Event::Paste(clipboard?)], + "edit.selectAll" => vec![key(egui::Key::A, false)], + "edit.undo" => vec![key(egui::Key::Z, false)], + "edit.redo" => vec![key(egui::Key::Z, true)], + _ => return None, + }) +} + +/// Run a native menu item (by id) the way the in-window menu bar does. While a text field has +/// the keyboard, Edit ▸ Cut/Copy/Paste/Select All/Undo/Redo edit the text instead. +pub fn activate(app: &mut EffectcraftApp, ctx: &egui::Context, menu: &NativeMenu, id: &str) -> Result { + let item = menu.find(id).ok_or_else(|| format!("no native menu item `{id}`"))?; + let (command, params) = (item.command.clone(), item.params.clone()); + if ctx.egui_wants_keyboard_input() { + let clip = if command == "edit.paste" { app.hooks.clipboard_text.as_ref().and_then(|f| f()) } else { None }; + if let Some(events) = text_field_event(&command, clip) { + app.synthetic.extend(events); + ctx.request_repaint(); + return Ok(Value::Null); + } + } + let params = if params.is_null() { json!({}) } else { params }; + let r = crate::menus::invoke(app, ctx, &command, params); + if let Err(e) = &r { + app.ui.status = e.clone(); + } + ctx.request_repaint(); + r +} + +#[cfg(test)] +mod tests { + use super::*; + + fn app() -> EffectcraftApp { + let mut s = effectcraft_engine::Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + EffectcraftApp::new(s) + } + + #[test] + fn every_engine_entry_is_in_the_native_menu() { + let app = app(); + let spec = build(&app); + fn commands(n: &[NativeNode], out: &mut Vec<(String, String)>) { + for x in n { + match x { + NativeNode::Item(i) if i.command != "file.openRecent" => out.push((i.command.clone(), i.params.to_string())), + NativeNode::Predefined { command, .. } if !command.is_empty() => out.push((command.clone(), "null".into())), + NativeNode::Submenu { children, .. } => commands(children, out), + _ => {} + } + } + } + let mut got = vec![]; + commands(&spec.menus, &mut got); + let want: Vec<(String, String)> = effectcraft_engine::menus::entries().into_iter().map(|(_, e)| (e.command.clone(), e.params.to_string())).collect(); + assert_eq!(got, want, "native menu must mirror the engine tree entry for entry"); + // Top-level menus in order. + let tops: Vec<&str> = spec + .menus + .iter() + .filter_map(|n| match n { + NativeNode::Submenu { label, .. } => Some(label.as_str()), + _ => None, + }) + .collect(); + assert_eq!(tops, effectcraft_engine::menus::top_level()); + // Ids are unique. + let ids: std::collections::BTreeSet<&str> = spec.items().iter().map(|i| i.id.as_str()).collect(); + assert_eq!(ids.len(), spec.items().len()); + // The spec serializes (agents can read it). + let v = serde_json::to_value(&spec).unwrap(); + assert!(v["menus"].as_array().is_some_and(|a| a.len() >= 9)); + } + + #[test] + fn hide_items_are_predefined_on_macos() { + let spec = build(&app()); + let mut roles = vec![]; + fn rec(n: &[NativeNode], out: &mut Vec) { + for x in n { + match x { + NativeNode::Predefined { role, .. } => out.push(*role), + NativeNode::Submenu { children, .. } => rec(children, out), + _ => {} + } + } + } + rec(&spec.menus, &mut roles); + if cfg!(target_os = "macos") { + assert_eq!(roles, vec![Role::Services, Role::Hide, Role::HideOthers, Role::ShowAll]); + // Quit and About dispatch our commands (clean shutdown, our About dialog). + assert!(spec.items().iter().any(|i| i.command == "app.quit" && i.accelerator.as_deref() == Some("Cmd+Q"))); + assert!(spec.items().iter().any(|i| i.command == "app.about")); + } else { + assert!(roles.is_empty()); + } + } + + #[test] + fn accelerators_convert() { + assert_eq!(accelerator("Cmd+Shift+S").as_deref(), Some("Cmd+Shift+S")); + assert_eq!(accelerator("Cmd+Alt+;").as_deref(), Some("Cmd+Alt+;")); + assert_eq!(accelerator("Cmd+`").as_deref(), Some("Cmd+`")); + assert_eq!(accelerator("Ctrl+Cmd+F").as_deref(), Some("Ctrl+Cmd+F")); + assert_eq!(accelerator("F3").as_deref(), Some("F3")); + assert_eq!(accelerator("Cmd+Alt+0").as_deref(), Some("Cmd+Alt+0")); + // Keys that would swallow typing stay window shortcuts. + assert_eq!(accelerator("V"), None); + assert_eq!(accelerator("Space"), None); + assert_eq!(accelerator("Shift+="), None); + assert_eq!(accelerator("Alt+["), None); + assert_eq!(accelerator("Num*"), None); + // Every menu shortcut either converts or is a single-key/Alt/Shift shortcut. + let app = app(); + for i in build(&app).items() { + if let Some(sc) = &i.shortcut { + let primary = sc.contains("Cmd") || sc.contains("Ctrl"); + if primary { + assert!(i.accelerator.is_some(), "{} ({sc}) has no accelerator", i.label); + } + } + } + } + + #[test] + fn state_changes_flow_into_the_spec() { + let mut app = app(); + let k0 = state_key(&app); + let spec0 = build(&app); + let undo = |s: &NativeMenu| s.items().into_iter().find(|i| i.command == "edit.undo").map(|i| (i.label.clone(), i.enabled)).unwrap(); + // Select a layer and toggle Lock: the key changes, the Lock entry gets a check mark. + let first = app.session.active_comp().unwrap().layers[0].id.0; + app.session.execute("layer.select", json!({"layers": [first]})).unwrap(); + app.session.execute("layer.setSwitch", json!({"switch": "lock"})).unwrap(); + assert_ne!(state_key(&app), k0); + let spec1 = build(&app); + assert_eq!(spec0.structure(), spec1.structure(), "state changes don't rebuild the menu"); + let lock = spec1.items().into_iter().find(|i| i.command == "layer.setSwitch" && i.params["switch"] == "lock").unwrap().checked; + assert_eq!(lock, Some(true)); + assert_ne!(undo(&spec0), undo(&spec1)); + // A recent project changes the structure (Open Recent grows). + app.session.prefs.push_recent("/tmp/Recent Test.ecproj"); + let spec2 = build(&app); + assert_ne!(spec1.structure(), spec2.structure()); + assert!(spec2.items().iter().any(|i| i.command == "file.openRecent" && i.label == "Recent Test.ecproj")); + } + + #[test] + fn activation_dispatches_the_menu_command() { + let mut app = app(); + let ctx = egui::Context::default(); + let spec = build(&app); + let id = spec.items().into_iter().find(|i| i.command == "layer.newNull").unwrap().id.clone(); + let n = app.session.active_comp().unwrap().layers.len(); + activate(&mut app, &ctx, &spec, &id).unwrap(); + assert_eq!(app.session.active_comp().unwrap().layers.len(), n + 1); + assert!(activate(&mut app, &ctx, &spec, "m.999").is_err()); + assert!(text_field_event("edit.copy", None).is_some()); + assert!(text_field_event("edit.paste", None).is_none()); + assert!(text_field_event("layer.newNull", None).is_none()); + } +} diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 000000000..41fb7204f --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 70d562376..20142225b 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -537,6 +537,35 @@ fn warp_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Roto Brush & Refine Edge: Freeze / Unfreeze and Propagate buttons and the segmentation status. +fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::roto as fx; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = fx::is_frozen(¶ms); + let d = fx::data(¶ms); + let running = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid) && app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, label, frozen, &t, egui::Id::new(("roto-freeze-ec", g.uid))).clicked() && !running { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Propagate", "roto.propagate") }; + if widgets::text_button(ui, b2, l2, false, &t, egui::Id::new(("roto-prop-ec", g.uid))).clicked() && !frozen { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.propagate", g.uid), b2, l2); + let status = match (app.session.roto_progress(), d.base) { + (Some(pr), _) if running => pr.banner(), + (_, None) => "No strokes: paint with the Roto Brush tool (Alt+W) in the Layer panel".to_string(), + (_, Some(b)) => format!("Base frame {b}, span {}–{}{}", d.span[0], d.span[1], if frozen { " (frozen)" } else { "" }), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -545,6 +574,8 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, + effectcraft_engine::effects::roto::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +951,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/flowchart.rs b/crates/ui-egui/src/panels/flowchart.rs new file mode 100644 index 000000000..2ba345cf2 --- /dev/null +++ b/crates/ui-egui/src/panels/flowchart.rs @@ -0,0 +1,460 @@ +//! The Flowchart panel (Composition ▸ Composition Flowchart): a node graph of a composition and +//! everything it uses — nested comps, footage, solids — optionally with its layers and their +//! effects. Left to right or top to bottom; double-click a comp to open it; Cmd/Ctrl-click a +//! node to reveal it in the Project panel (layers: select the layer); drag to pan. +//! +//! [`build`] is pure and cycle-safe (a comp nested in itself is drawn once, with an edge back), +//! so agents get the same graph through `flowchart.graph`. + +use std::collections::{BTreeMap, BTreeSet}; + +use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, Project}; +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::icons::{self, Icon}; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// Flowchart panel options (the panel's switches). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FlowOptions { + /// Show layers between a comp and its sources. + pub layers: bool, + /// Show each layer's effects (with layers shown). + pub effects: bool, + /// Show solids. + pub solids: bool, + /// Top to bottom instead of left to right. + pub vertical: bool, + /// The comp the chart starts from (None = the active comp). + pub root: Option, + #[serde(skip)] + pub pan: [f32; 2], + /// Selected node id. + #[serde(skip)] + pub selected: Option, +} + +impl Default for FlowOptions { + fn default() -> Self { + FlowOptions { layers: false, effects: false, solids: true, vertical: false, root: None, pan: [0.0, 0.0], selected: None } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FlowNode { + /// `item:`, `layer::` or `fx:::`. + pub id: String, + /// `comp`, `footage`, `solid`, `layer` or `effect`. + pub kind: &'static str, + pub label: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub item: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub comp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub layer: Option, + /// Distance from the root (column, or row when vertical). + pub depth: usize, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FlowGraph { + pub nodes: Vec, + /// (from, to) node ids: from a comp to what it uses. + pub edges: Vec<(String, String)>, +} + +/// Build the flowchart of `root` (a comp). Shared items appear once; cycles stop at the comp +/// already on the path. +pub fn build(project: &Project, root: ItemId, opts: &FlowOptions) -> FlowGraph { + struct B<'a> { + p: &'a Project, + o: &'a FlowOptions, + nodes: Vec, + index: BTreeMap, + edges: Vec<(String, String)>, + stack: BTreeSet, + } + impl B<'_> { + fn node(&mut self, n: FlowNode) -> String { + let id = n.id.clone(); + match self.index.get(&id) { + Some(&i) => self.nodes[i].depth = self.nodes[i].depth.min(n.depth), + None => { + self.index.insert(id.clone(), self.nodes.len()); + self.nodes.push(n); + } + } + id + } + fn edge(&mut self, a: &str, b: &str) { + let e = (a.to_string(), b.to_string()); + if !self.edges.contains(&e) { + self.edges.push(e); + } + } + fn item(&mut self, id: ItemId, depth: usize) -> Option { + let it = self.p.item(id)?; + let kind = match &it.kind { + ItemKind::Comp(_) => "comp", + ItemKind::Footage(_) => "footage", + ItemKind::Solid(_) => "solid", + ItemKind::Folder => return None, + }; + if kind == "solid" && !self.o.solids { + return None; + } + let seen = self.index.contains_key(&format!("item:{}", id.0)); + let nid = self.node(FlowNode { id: format!("item:{}", id.0), kind, label: it.name.clone(), item: Some(id.0), comp: None, layer: None, depth }); + if kind == "comp" && !seen && !self.stack.contains(&id.0) { + self.comp(id, depth); + } + Some(nid) + } + fn comp(&mut self, cid: ItemId, depth: usize) { + let Some(c) = self.p.comp(cid) else { return }; + self.stack.insert(cid.0); + let from = format!("item:{}", cid.0); + for l in &c.layers { + let src_depth = if self.o.layers { depth + 2 } else { depth + 1 }; + let mut parent = from.clone(); + if self.o.layers { + let lid = self.node(FlowNode { + id: format!("layer:{}:{}", cid.0, l.id.0), + kind: "layer", + label: l.name.clone(), + item: None, + comp: Some(cid.0), + layer: Some(l.id.0), + depth: depth + 1, + }); + self.edge(&from, &lid); + if self.o.effects + && let Some(fx) = l.effects() + { + for g in fx.groups() { + let fid = self.node(FlowNode { + id: format!("fx:{}:{}:{}", cid.0, l.id.0, g.uid), + kind: "effect", + label: g.name.clone(), + item: None, + comp: Some(cid.0), + layer: Some(l.id.0), + depth: depth + 2, + }); + self.edge(&lid, &fid); + } + } + parent = lid; + } + let src = match l.source { + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => Some(item), + _ => None, + }; + if let Some(item) = src { + // A comp already on the path: link back to it, don't descend (cycle). + if self.stack.contains(&item.0) { + self.edge(&parent, &format!("item:{}", item.0)); + } else if let Some(sid) = self.item(item, src_depth) { + self.edge(&parent, &sid); + } + } + } + self.stack.remove(&cid.0); + } + } + let mut b = B { p: project, o: opts, nodes: vec![], index: BTreeMap::new(), edges: vec![], stack: BTreeSet::new() }; + b.item(root, 0); + FlowGraph { nodes: b.nodes, edges: b.edges } +} + +const NODE_W: f32 = 168.0; +const NODE_H: f32 = 34.0; + +/// Node positions (top-left, before panning): one column (or row) per depth, in discovery order. +pub fn layout(g: &FlowGraph, vertical: bool) -> Vec<[f32; 2]> { + let mut slot: BTreeMap = BTreeMap::new(); + g.nodes + .iter() + .map(|n| { + let k = slot.entry(n.depth).or_insert(0); + let i = *k as f32; + *k += 1; + let d = n.depth as f32; + if vertical { [i * (NODE_W + 24.0), d * (NODE_H + 46.0)] } else { [d * (NODE_W + 60.0), i * (NODE_H + 14.0)] } + }) + .collect() +} + +/// The comp the panel charts. +pub fn root(app: &EffectcraftApp) -> Option { + app.ui.flowchart.root.map(ItemId).filter(|r| app.session.project.comp(*r).is_some()).or_else(|| app.session.active_comp_id()) +} + +/// `flowchart.options` / `flowchart.graph` (UI commands). +pub fn command(app: &mut EffectcraftApp, id: &str, p: &Value) -> Result { + let o = &mut app.ui.flowchart; + let b = |k: &str| p.get(k).and_then(Value::as_bool); + if let Some(v) = b("layers") { + o.layers = v; + } + if let Some(v) = b("effects") { + o.effects = v; + } + if let Some(v) = b("solids") { + o.solids = v; + } + if let Some(d) = p.get("direction").and_then(Value::as_str) { + o.vertical = match d { + "lr" | "leftToRight" => false, + "tb" | "topToBottom" => true, + _ => return Err(format!("direction: lr|tb, not `{d}`")), + }; + } + if let Some(c) = p.get("comp") { + let cid = match c { + Value::Number(n) => n.as_u64().map(ItemId), + Value::String(s) => app.session.project.find_by_name(s).map(|i| i.id), + _ => None, + }; + let cid = cid.filter(|c| app.session.project.comp(*c).is_some()).ok_or("comp: a composition id or name")?; + app.ui.flowchart.root = Some(cid.0); + } + let o = app.ui.flowchart.clone(); + if id == "flowchart.graph" { + let r = root(app).ok_or("no composition")?; + return Ok(serde_json::to_value(build(&app.session.project, r, &o)).unwrap_or_default()); + } + Ok(json!({"layers": o.layers, "effects": o.effects, "solids": o.solids, "direction": if o.vertical { "tb" } else { "lr" }, "root": o.root})) +} + +fn kind_style(kind: &str) -> (Icon, Color32) { + match kind { + "comp" => (Icon::Comp, Color32::from_rgb(0x3d, 0x5a, 0x8c)), + "footage" => (Icon::Footage, Color32::from_rgb(0x3c, 0x6e, 0x58)), + "solid" => (Icon::Solid, Color32::from_rgb(0x6a, 0x58, 0x3a)), + "effect" => (Icon::Fx, Color32::from_rgb(0x6c, 0x44, 0x6e)), + _ => (Icon::Null, Color32::from_rgb(0x44, 0x48, 0x52)), + } +} + +/// Draw the Flowchart panel. +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.tl_bg); + let Some(rid) = root(app) else { + p.text(rect.center(), Align2::CENTER_CENTER, "(no composition)", Tokens::ui(12.0), t.text_faint); + return; + }; + // ---- toolbar: switches and direction. + let bar = Rect::from_min_size(rect.min, vec2(rect.width(), 30.0)); + p.rect_filled(bar, 0.0, t.panel_bg); + let root_name = app.session.project.item(rid).map(|i| i.name.clone()).unwrap_or_default(); + p.text(pos2(bar.min.x + 10.0, bar.center().y), Align2::LEFT_CENTER, format!("Flowchart: {root_name}"), Tokens::medium(12.0), t.text); + let mut x = bar.max.x - 8.0; + let mut cmds: Vec<(String, Value)> = vec![]; + let o = app.ui.flowchart.clone(); + for (key, label, on) in [ + ("direction", if o.vertical { "Top to Bottom" } else { "Left to Right" }, false), + ("effects", "Effects", o.effects), + ("layers", "Layers", o.layers), + ("solids", "Solids", o.solids), + ] { + let w = 18.0 + label.len() as f32 * 6.6; + let r = Rect::from_min_size(pos2(x - w, bar.min.y + 5.0), vec2(w, 20.0)); + x -= w + 6.0; + if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("flow-opt", key))).clicked() { + let v = if key == "direction" { json!({"direction": if o.vertical { "lr" } else { "tb" }}) } else { json!({key: !on}) }; + cmds.push(("flowchart.options".into(), v)); + } + app.auto.add(&format!("flowchart.option.{key}"), r, label); + } + + // ---- graph. + let body = Rect::from_min_max(pos2(rect.min.x, bar.max.y), rect.max); + let g = build(&app.session.project, rid, &o); + let pos = layout(&g, o.vertical); + let origin = body.min + vec2(24.0 + o.pan[0], 24.0 + o.pan[1]); + let bg = ui.interact(body, egui::Id::new("flow-bg"), Sense::click_and_drag()); + if bg.dragged() { + let d = bg.drag_delta(); + app.ui.flowchart.pan[0] += d.x; + app.ui.flowchart.pan[1] += d.y; + } + if ui.rect_contains_pointer(body) { + let s = ui.input(|i| i.smooth_scroll_delta); + if s != egui::Vec2::ZERO { + app.ui.flowchart.pan[0] += s.x; + app.ui.flowchart.pan[1] += s.y; + } + } + if bg.clicked() { + app.ui.flowchart.selected = None; + } + let gp = p.with_clip_rect(body); + let rect_of = |i: usize| Rect::from_min_size(origin + vec2(pos[i][0], pos[i][1]), vec2(NODE_W, NODE_H)); + let idx: BTreeMap<&str, usize> = g.nodes.iter().enumerate().map(|(i, n)| (n.id.as_str(), i)).collect(); + // Edges first (curves with an arrowhead at the target). + for (a, b) in &g.edges { + let (Some(&ia), Some(&ib)) = (idx.get(a.as_str()), idx.get(b.as_str())) else { continue }; + let (ra, rb) = (rect_of(ia), rect_of(ib)); + let back = g.nodes[ib].depth <= g.nodes[ia].depth; + let (s, e, n) = if o.vertical { (ra.center_bottom(), rb.center_top(), vec2(0.0, 1.0)) } else { (ra.right_center(), rb.left_center(), vec2(1.0, 0.0)) }; + let k = if o.vertical { (e.y - s.y).abs() * 0.5 } else { (e.x - s.x).abs() * 0.5 }.max(30.0); + let col = if back { Color32::from_rgb(0xd0, 0x80, 0x40) } else { t.text_faint }; + let curve = egui::epaint::CubicBezierShape::from_points_stroke([s, s + n * k, e - n * k, e], false, Color32::TRANSPARENT, Stroke::new(1.2, col)); + gp.add(curve); + let (l, r) = if o.vertical { (vec2(-4.0, -7.0), vec2(4.0, -7.0)) } else { (vec2(-7.0, -4.0), vec2(-7.0, 4.0)) }; + gp.add(egui::Shape::convex_polygon(vec![e, e + l, e + r], col, Stroke::NONE)); + } + let cmd_down = ui.input(|i| i.modifiers.command || i.modifiers.ctrl); + for (i, n) in g.nodes.iter().enumerate() { + let r = rect_of(i); + if !r.intersects(body) { + continue; + } + let (icon, fill) = kind_style(n.kind); + let selected = o.selected.as_deref() == Some(n.id.as_str()); + let resp = ui.interact(r.intersect(body), egui::Id::new(("flow-node", &n.id)), Sense::click()); + let fill = if resp.hovered() { fill.gamma_multiply(1.25) } else { fill }; + gp.rect_filled(r, 5.0, fill); + gp.rect_stroke( + r, + 5.0, + Stroke::new(if selected { 2.0 } else { 1.0 }, if selected { t.accent } else { Color32::from_black_alpha(120) }), + StrokeKind::Inside, + ); + icons::paint(&gp, Rect::from_center_size(pos2(r.min.x + 16.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); + gp.with_clip_rect(r.shrink(2.0).intersect(body)).text( + pos2(r.min.x + 30.0, r.center().y), + Align2::LEFT_CENTER, + &n.label, + Tokens::ui(12.0), + Color32::WHITE, + ); + app.auto.add(&format!("flowchart.node.{}", n.id), r, &n.label); + if resp.clicked() { + app.ui.flowchart.selected = Some(n.id.clone()); + if cmd_down { + // Reveal: items in the Project panel, layers selected in their comp. + match (n.item, n.comp, n.layer) { + (Some(item), ..) => { + cmds.push(("project.select".into(), json!({"items": [item]}))); + cmds.push(("window.panel".into(), json!({"panel": "project"}))); + } + (None, Some(c), Some(l)) => { + cmds.push(("comp.open".into(), json!({"comp": c}))); + cmds.push(("layer.select".into(), json!({"layers": [l]}))); + } + _ => {} + } + } + } + if resp.double_clicked() { + match (n.kind, n.item, n.comp) { + ("comp", Some(c), _) => cmds.push(("comp.open".into(), json!({"comp": c}))), + (_, _, Some(c)) => cmds.push(("comp.open".into(), json!({"comp": c}))), + _ => {} + } + } + resp.on_hover_text(format!("{} — double-click to open, {}-click to reveal", n.label, if cfg!(target_os = "macos") { "Cmd" } else { "Ctrl" })); + } + for (id, params) in cmds { + if let Err(e) = crate::menus::invoke(app, &ctx, &id, params) { + app.ui.status = e; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use effectcraft_engine::Session; + + /// Main ⊃ Mid ⊃ Inner (with a solid), Main also uses Inner directly. + fn session() -> (Session, ItemId, ItemId, ItemId) { + let mut s = Session::default(); + let id = |v: Value| ItemId(v["id"].as_u64().or_else(|| v["comp"].as_u64()).or_else(|| v["item"].as_u64()).unwrap()); + let inner = id(s.execute("comp.new", json!({"name": "Inner", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.newSolid", json!({"name": "Red", "color": "#ff0000", "width": 100, "height": 100})).unwrap(); + s.execute("layer.select", json!({"layers": []})).unwrap(); + let mid = id(s.execute("comp.new", json!({"name": "Mid", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + let main = id(s.execute("comp.new", json!({"name": "Main", "width": 100, "height": 100, "duration": 2})).unwrap()); + s.execute("layer.addItem", json!({"item": mid.0})).unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + s.execute("layer.newText", json!({"text": "Title"})).unwrap(); + (s, main, mid, inner) + } + + #[test] + fn nested_comps_make_a_graph() { + let (s, main, mid, inner) = session(); + let g = build(&s.project, main, &FlowOptions::default()); + let label = |id: &str| g.nodes.iter().find(|n| n.id == id).map(|n| (n.label.clone(), n.depth)); + assert_eq!(label(&format!("item:{}", main.0)), Some(("Main".into(), 0))); + assert_eq!(label(&format!("item:{}", mid.0)), Some(("Mid".into(), 1))); + // Inner is reached at depth 1 (directly) and 2 (through Mid): drawn once, at depth 1. + assert_eq!(label(&format!("item:{}", inner.0)), Some(("Inner".into(), 1))); + assert_eq!(g.nodes.iter().filter(|n| n.label == "Inner").count(), 1); + let has = |a: ItemId, b: ItemId| g.edges.contains(&(format!("item:{}", a.0), format!("item:{}", b.0))); + assert!(has(main, mid) && has(mid, inner) && has(main, inner)); + // The solid is shown (Solids on) and hidden with Solids off; text layers have no source. + assert!(g.nodes.iter().any(|n| n.kind == "solid")); + let g2 = build(&s.project, main, &FlowOptions { solids: false, ..Default::default() }); + assert!(!g2.nodes.iter().any(|n| n.kind == "solid")); + // Layers between comps and sources; effects under layers. + let g3 = build(&s.project, main, &FlowOptions { layers: true, ..Default::default() }); + assert!(g3.nodes.iter().any(|n| n.kind == "layer" && n.label == "Title")); + let mid_layer = g3.nodes.iter().find(|n| n.kind == "layer" && n.comp == Some(main.0) && n.label == "Mid").unwrap(); + assert!(g3.edges.contains(&(mid_layer.id.clone(), format!("item:{}", mid.0)))); + // Layout: one column per depth, no overlaps. + let pos = layout(&g3, false); + for i in 0..pos.len() { + for j in i + 1..pos.len() { + assert!(pos[i] != pos[j], "{} and {} overlap", g3.nodes[i].label, g3.nodes[j].label); + } + } + } + + #[test] + fn cycles_are_safe() { + let (mut s, main, mid, _) = session(); + // Make Mid contain Main: Main → Mid → Main. + s.execute("comp.open", json!({"comp": mid.0})).unwrap(); + // Adding a cyclic layer may be refused by the engine; build a cycle in the model directly. + let mut p = (*s.project).clone(); + let mut c = (*p.comp(mid).unwrap()).clone(); + let mut l = c.layers[0].clone(); + l.id = effectcraft_engine::project::LayerId(9999); + l.source = LayerSource::Comp { item: main }; + c.layers.push(l); + if let Some(ItemKind::Comp(arc)) = p.items.get_mut(&mid).map(|i| &mut i.kind) { + *arc = std::sync::Arc::new(c); + } + let g = build(&p, main, &FlowOptions { layers: true, effects: true, ..Default::default() }); + assert_eq!(g.nodes.iter().filter(|n| n.kind == "comp" && n.label == "Main").count(), 1); + // The back edge points at the root. + assert!(g.edges.iter().any(|(_, b)| *b == format!("item:{}", main.0))); + assert!(g.nodes.len() < 50); + } + + #[test] + fn options_command() { + let (s, main, ..) = session(); + let mut app = EffectcraftApp::new(s); + let r = command(&mut app, "flowchart.options", &json!({"layers": true, "direction": "tb"})).unwrap(); + assert_eq!(r["direction"], "tb"); + assert!(app.ui.flowchart.layers && app.ui.flowchart.vertical); + assert!(command(&mut app, "flowchart.options", &json!({"direction": "diagonal"})).is_err()); + let g = command(&mut app, "flowchart.graph", &json!({"comp": "Main"})).unwrap(); + assert_eq!(g["nodes"][0]["id"], format!("item:{}", main.0)); + assert!(g["edges"].as_array().is_some_and(|e| !e.is_empty())); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index af117e8eb..3e0519481 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -108,6 +108,16 @@ fn has(p: &Value, keys: &[&str]) -> bool { } /// If `id` is a dialog command invoked without its parameters, open its form and return true. +/// The import includes a Photoshop document. +fn is_layered(p: &Value) -> bool { + let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb"))); + match p.get("paths").or(p.get("path")) { + Some(Value::Array(a)) => a.iter().any(psd), + Some(v) => psd(v), + None => false, + } +} + pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let s = &app.session; let comp = s.active_comp(); @@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t); let base = p.clone(); let (title, fields): (String, Vec) = match id { + // Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes). + "file.import" if !has(p, &["importAs"]) && is_layered(p) => ( + "Import Photoshop File".into(), + vec![Field::choice( + "importAs", + "Import Kind", + &[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))], + 1, + )], + ), "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/home.rs b/crates/ui-egui/src/panels/home.rs new file mode 100644 index 000000000..42ded4b74 --- /dev/null +++ b/crates/ui-egui/src/panels/home.rs @@ -0,0 +1,389 @@ +//! The Home screen (shown in the Composition panel at launch, from the header's Home button, and +//! by the Learn workspace): New Project / Open Project / New Composition / Open Demo, the recent +//! projects (File ▸ Open Recent, from Settings) with thumbnails, folders and dates, and where +//! After Effects has its Learn / What's New area, the ArtCraft community links. +//! +//! Thumbnails: when a project is opened or saved and the viewer has its frame, a 96×54 RGB +//! thumbnail is stored in the config store (`thumb-.txt`, hex) and shown here. + +use std::hash::{Hash, Hasher}; + +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde::Serialize; +use serde_json::json; + +use crate::icons::{self, Icon}; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +pub const THUMB_W: usize = 96; +pub const THUMB_H: usize = 54; + +/// One row of the recent projects list. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct RecentEntry { + pub index: usize, + pub path: String, + pub name: String, + pub folder: String, + /// Last modified (`2026-10-02 14:05`, UTC), when the file is readable. + pub modified: Option, + pub exists: bool, +} + +/// The recent projects, most recent first (Settings ▸ General ▸ recent items). +pub fn recent_entries(prefs: &effectcraft_engine::prefs::Prefs) -> Vec { + prefs + .recent_projects + .iter() + .enumerate() + .map(|(index, p)| { + let path = std::path::Path::new(p); + let meta = std::fs::metadata(path).ok(); + let modified = meta + .as_ref() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| format_utc(d.as_secs() as i64)); + RecentEntry { + index, + path: p.clone(), + name: path.file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| p.clone()), + folder: path.parent().map(|d| d.to_string_lossy().to_string()).unwrap_or_default(), + modified, + exists: meta.is_some(), + } + }) + .collect() +} + +/// `YYYY-MM-DD HH:MM` (UTC) from Unix seconds. +pub fn format_utc(secs: i64) -> String { + let days = secs.div_euclid(86_400); + let rem = secs.rem_euclid(86_400); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02} {:02}:{:02}", rem / 3600, (rem % 3600) / 60) +} + +/// The config-store name of a project's thumbnail. +pub fn thumb_name(path: &str) -> String { + let mut h = std::collections::hash_map::DefaultHasher::new(); + path.hash(&mut h); + format!("thumb-{:016x}.txt", h.finish()) +} + +/// Downscale an image to a letterboxed THUMB_W×THUMB_H RGB thumbnail (hex text). +pub fn encode_thumb(img: &egui::ColorImage) -> String { + let [w, h] = img.size; + let mut out = String::with_capacity(THUMB_W * THUMB_H * 6); + let k = (THUMB_W as f32 / w.max(1) as f32).min(THUMB_H as f32 / h.max(1) as f32); + let (dw, dh) = (w as f32 * k, h as f32 * k); + let (ox, oy) = ((THUMB_W as f32 - dw) / 2.0, (THUMB_H as f32 - dh) / 2.0); + for y in 0..THUMB_H { + for x in 0..THUMB_W { + let (fx, fy) = ((x as f32 + 0.5 - ox) / k, (y as f32 + 0.5 - oy) / k); + let c = if fx >= 0.0 && fy >= 0.0 && (fx as usize) < w && (fy as usize) < h { + let p = img.pixels[fy as usize * w + fx as usize]; + [p.r(), p.g(), p.b()] + } else { + [0x16, 0x17, 0x1c] + }; + for v in c { + out.push_str(&format!("{v:02x}")); + } + } + } + out +} + +pub fn decode_thumb(text: &str) -> Option { + let t = text.trim(); + if t.len() != THUMB_W * THUMB_H * 6 { + return None; + } + let b: Vec = (0..t.len()).step_by(2).map(|i| u8::from_str_radix(&t[i..i + 2], 16)).collect::>().ok()?; + let px = b.chunks_exact(3).map(|c| Color32::from_rgb(c[0], c[1], c[2])).collect(); + Some(egui::ColorImage::new([THUMB_W, THUMB_H], px)) +} + +/// Store the viewer's frame as the open project's thumbnail after it is opened or saved (once per +/// saved revision, when the viewer shows that revision). +pub fn capture_thumbnail(app: &mut EffectcraftApp) { + let s = &app.session; + let (Some(path), Some(_)) = (s.path.clone(), s.config.as_ref()) else { return }; + if s.is_dirty() { + return; + } + let key = (path.clone(), s.saved_revision); + if app.home_thumb_saved.as_ref() == Some(&key) { + return; + } + if app.viewer_shown.as_ref().map(|(_, k)| k.revision) != Some(s.revision) { + return; + } + let Some(img) = app.viewer_pixels() else { return }; + let text = encode_thumb(&img); + if let Some(c) = &app.session.config + && let Err(e) = c.write(&thumb_name(&path), &text) + { + log::warn!("project thumbnail: {e}"); + } + app.home_thumbs.remove(&path); + app.home_thumb_saved = Some(key); +} + +fn thumb_texture(app: &mut EffectcraftApp, ctx: &egui::Context, path: &str) -> Option { + if let Some(t) = app.home_thumbs.get(path) { + return t.clone(); + } + let tex = app + .session + .config + .as_ref() + .and_then(|c| c.read(&thumb_name(path))) + .and_then(|t| decode_thumb(&t)) + .map(|img| ctx.load_texture(format!("home-thumb-{path}"), img, egui::TextureOptions::LINEAR)); + app.home_thumbs.insert(path.to_string(), tex.clone()); + tex +} + +/// Draw the Home screen. +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, Color32::from_rgb(0x1b, 0x1d, 0x24)); + let wide = rect.width() >= 820.0; + let pad = 32.0; + let left_w = if wide { 250.0 } else { rect.width() - pad * 2.0 }; + let mut actions: Vec<(&str, serde_json::Value)> = vec![]; + + // ---- left: brand + start actions. + let x0 = rect.min.x + pad; + let mut y = rect.min.y + pad; + crate::header::paint_logo(&p, Rect::from_min_size(pos2(x0, y), vec2(40.0, 40.0))); + p.text(pos2(x0 + 52.0, y + 12.0), Align2::LEFT_CENTER, "EffectCraft", Tokens::semibold(22.0), Color32::WHITE); + p.text(pos2(x0 + 52.0, y + 32.0), Align2::LEFT_CENTER, format!("Version {}", env!("CARGO_PKG_VERSION")), Tokens::ui(11.0), t.text_faint); + y += 64.0; + for (label, id, primary) in [ + ("New Project", "file.newProject", true), + ("Open Project…", "file.open", false), + ("New Composition", "app.newComp", false), + ("Open Demo Project", "file.openDemoProject", false), + ("Import Footage…", "file.import", false), + ] { + let r = Rect::from_min_size(pos2(x0, y), vec2(left_w.min(250.0), 32.0)); + if widgets::text_button(ui, r, label, primary, &t, egui::Id::new(("home", id))).clicked() { + actions.push((id, json!({}))); + } + app.auto.add(&format!("home.{id}"), r, label); + y += 40.0; + } + + // ---- community (After Effects' Learn / What's New area). + let (cx, mut cy, cw) = if wide { (x0, y + 18.0, left_w) } else { (x0, y + 10.0, left_w) }; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "Community", Tokens::semibold(13.0), t.text); + cy += 16.0; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "Tutorials, help and what's new", Tokens::ui(11.0), t.text_faint); + cy += 16.0; + let links = [ + (Icon::Chat, "Join the ArtCraft Discord", "help.discord"), + (Icon::Globe, "getartcraft.com", "help.website"), + (Icon::Globe, "EffectCraft home page", "help.appPage"), + (Icon::Code, "EffectCraft on GitHub", "help.github"), + ]; + for (icon, label, cmd) in links { + let r = Rect::from_min_size(pos2(cx, cy), vec2(cw.min(250.0), 30.0)); + let resp = ui.interact(r, egui::Id::new(("home-link", cmd)), Sense::click()); + let discord = cmd == "help.discord"; + let bg = if discord { + Color32::from_rgb(0x58, 0x65, 0xf2) + } else if resp.hovered() { + t.hover + } else { + Color32::from_rgb(0x2a, 0x2c, 0x34) + }; + p.rect_filled(r, 15.0, bg); + icons::paint(&p, Rect::from_center_size(pos2(r.min.x + 20.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); + p.text(pos2(r.min.x + 36.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::medium(12.0), Color32::WHITE); + app.auto.add(&format!("home.{cmd}"), r, label); + if resp.clicked() { + let _ = app.session.execute(cmd, json!({})); + } + cy += 36.0; + } + cy += 8.0; + p.text(pos2(cx, cy), Align2::LEFT_CENTER, "More ArtCraft apps", Tokens::ui(11.0), t.text_faint); + cy += 12.0; + let sib = effectcraft_engine::links::SIBLINGS; + let per_row = 2usize; + let sw = (cw.min(250.0) - 6.0) / per_row as f32; + for (i, (name, slug)) in sib.iter().enumerate() { + let r = Rect::from_min_size(pos2(cx + (i % per_row) as f32 * (sw + 6.0), cy + (i / per_row) as f32 * 28.0), vec2(sw, 24.0)); + let resp = ui.interact(r, egui::Id::new(("sib", *slug)), Sense::click()); + p.rect_filled(r, 12.0, if resp.hovered() { t.hover } else { Color32::from_rgb(0x24, 0x26, 0x2e) }); + p.text(r.center(), Align2::CENTER_CENTER, *name, Tokens::ui(11.5), t.text); + app.auto.add(&format!("home.sibling.{slug}"), r, name); + if resp.clicked() { + let _ = app.session.execute("help.sibling", json!({"app": slug})); + } + } + let community_bottom = cy + sib.len().div_ceil(per_row) as f32 * 28.0; + + // ---- recent projects. + let (rx, mut ry, rw) = if wide { + let rx = x0 + left_w + 40.0; + (rx, rect.min.y + pad + 6.0, rect.max.x - pad - rx) + } else { + (x0, community_bottom + 24.0, left_w) + }; + p.text(pos2(rx, ry + 6.0), Align2::LEFT_CENTER, "Recent", Tokens::semibold(16.0), Color32::WHITE); + let entries = recent_entries(&app.session.prefs); + if !entries.is_empty() { + let cr = Rect::from_min_size(pos2(rx + rw - 100.0, ry - 4.0), vec2(100.0, 20.0)); + let resp = ui.interact(cr, egui::Id::new("home-clear-recent"), Sense::click()); + p.text(cr.right_center(), Align2::RIGHT_CENTER, "Clear list", Tokens::ui(11.0), if resp.hovered() { t.text } else { t.text_faint }); + app.auto.add("home.clearRecent", cr, "Clear Recent Projects"); + if resp.clicked() { + actions.push(("file.clearRecent", json!({}))); + } + } + ry += 26.0; + // Column headings. + let date_w = 130.0; + p.text(pos2(rx + 112.0, ry), Align2::LEFT_CENTER, "Name", Tokens::ui(11.0), t.text_faint); + p.text(pos2(rx + rw - date_w, ry), Align2::LEFT_CENTER, "Modified", Tokens::ui(11.0), t.text_faint); + ry += 10.0; + p.line_segment([pos2(rx, ry), pos2(rx + rw, ry)], Stroke::new(1.0, t.separator)); + ry += 6.0; + if entries.is_empty() { + p.text(pos2(rx, ry + 20.0), Align2::LEFT_CENTER, "No recent projects. Projects you open or save appear here.", Tokens::ui(12.0), t.text_faint); + } + let row_h = 66.0; + for e in &entries { + let r = Rect::from_min_size(pos2(rx, ry), vec2(rw, row_h - 4.0)); + if r.min.y > rect.max.y { + break; + } + let resp = ui.interact(r, egui::Id::new(("home-recent", e.index)), Sense::click()).on_hover_text(&e.path); + if resp.hovered() { + p.rect_filled(r, 6.0, t.hover); + } + let tr = Rect::from_min_size(pos2(r.min.x + 6.0, r.min.y + 4.0), vec2(THUMB_W as f32, THUMB_H as f32)); + match thumb_texture(app, &ctx, &e.path) { + Some(tex) => { + p.image(tex.id(), tr, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } + None => { + p.rect_filled(tr, 3.0, Color32::from_rgb(0x24, 0x26, 0x2e)); + icons::paint(&p, Rect::from_center_size(tr.center(), vec2(20.0, 20.0)), Icon::Comp, t.text_faint); + } + } + p.rect_stroke(tr, 3.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); + let tx = tr.max.x + 12.0; + let text_clip = p.with_clip_rect(Rect::from_min_max(pos2(tx, r.min.y), pos2(r.max.x - date_w - 8.0, r.max.y))); + text_clip.text(pos2(tx, r.min.y + 20.0), Align2::LEFT_CENTER, &e.name, Tokens::semibold(13.0), if e.exists { Color32::WHITE } else { t.text_dim }); + let sub = if e.exists { e.folder.clone() } else { format!("{} (missing)", e.folder) }; + text_clip.text(pos2(tx, r.min.y + 40.0), Align2::LEFT_CENTER, sub, Tokens::ui(11.0), t.text_faint); + if let Some(d) = &e.modified { + p.text(pos2(r.max.x - date_w, r.min.y + 20.0), Align2::LEFT_CENTER, d, Tokens::ui(11.0), t.text_dim); + } + app.auto.add(&format!("home.recent.{}", e.index), r, &e.path); + if resp.clicked() { + actions.push(("file.openRecent", json!({"index": e.index}))); + } + ry += row_h; + } + + for (id, params) in actions { + let before = (app.session.path.clone(), app.session.revision); + match crate::menus::invoke(app, &ctx, id, params) { + Err(e) => app.ui.status = e, + // Leave Home unless only the list changed or a file dialog was cancelled. + Ok(_) => { + let cancelled = matches!(id, "file.open" | "file.import") && (app.session.path.clone(), app.session.revision) == before; + if id != "file.clearRecent" && !cancelled { + app.ui.start_screen = false; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recent_list_comes_from_settings() { + let mut prefs = effectcraft_engine::prefs::Prefs::default(); + prefs.push_recent("/projects/a/Alpha.ecproj"); + prefs.push_recent("/projects/b/Beta.ecproj"); + let e = recent_entries(&prefs); + assert_eq!(e.len(), 2); + assert_eq!((e[0].name.as_str(), e[0].folder.as_str(), e[0].index), ("Beta", "/projects/b", 0)); + assert_eq!(e[1].name, "Alpha"); + assert!(!e[0].exists && e[0].modified.is_none()); + // A real file has a date. + let dir = std::env::temp_dir().join(format!("ec-home-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let f = dir.join("Real.ecproj"); + std::fs::write(&f, "{}").unwrap(); + prefs.push_recent(&f.to_string_lossy()); + let e = recent_entries(&prefs); + assert!(e[0].exists && e[0].modified.as_ref().is_some_and(|d| d.len() == 16), "{:?}", e[0]); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn dates_and_thumbnails() { + assert_eq!(format_utc(0), "1970-01-01 00:00"); + assert_eq!(format_utc(1_790_000_000), "2026-09-21 14:13"); + assert_eq!(format_utc(951_782_400), "2000-02-29 00:00"); + let img = egui::ColorImage::new([200, 100], vec![Color32::from_rgb(10, 200, 30); 200 * 100]); + let text = encode_thumb(&img); + let back = decode_thumb(&text).unwrap(); + assert_eq!(back.size, [THUMB_W, THUMB_H]); + // Letterboxed: the middle is the image, the top row is the background (2:1 into 16:9). + assert_eq!(back.pixels[THUMB_H / 2 * THUMB_W + THUMB_W / 2], Color32::from_rgb(10, 200, 30)); + assert_ne!(back.pixels[THUMB_W / 2], Color32::from_rgb(10, 200, 30)); + assert!(decode_thumb("abc").is_none()); + assert_ne!(thumb_name("/a.ecproj"), thumb_name("/b.ecproj")); + } + + #[test] + fn home_screen_registers_its_controls() { + let mut s = effectcraft_engine::Session::default(); + s.prefs.push_recent("/projects/Alpha.ecproj"); + let mut app = EffectcraftApp::new(s); + let ctx = egui::Context::default(); + crate::theme::install(&ctx, &app.tokens); + ctx.run_ui(Default::default(), |_| {}).textures_delta.clear(); + ctx.run_ui(Default::default(), |ui| { + app.auto.begin_frame(); + show(&mut app, ui, Rect::from_min_size(pos2(0.0, 0.0), vec2(1200.0, 800.0))); + }) + .textures_delta + .clear(); + for id in [ + "home.file.newProject", + "home.file.open", + "home.app.newComp", + "home.file.openDemoProject", + "home.help.discord", + "home.help.github", + "home.recent.0", + ] { + assert!(app.auto.find(id).is_some(), "{id}"); + } + } +} diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index 5149241e8..20ccf3580 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -3,9 +3,13 @@ //! //! The image is the layer's source and masks followed by its effects up to the View menu's //! choice (by default the last Paint effect, as After Effects shows "Paint"). Dragging with a -//! paint tool records the stroke's points in layer space and, on release, runs `paint.stroke` +//! paint tools records the stroke's points in layer space and, on release, runs `paint.stroke` //! at the current time (the drag's duration feeds Write On). Alt/Option-click with the Clone //! Stamp sets the clone source. +//! +//! With a Roto Brush & Refine Edge effect the panel shows the effect's input with the matte +//! overlay, the segmentation span bar and the view / Freeze buttons; the Roto Brush and Refine +//! Edge tools paint their strokes here (see [`super::roto_tool`]). use effectcraft_engine::effects::paint; use effectcraft_engine::project::{Layer, LayerId}; @@ -46,9 +50,15 @@ pub fn current_layer(app: &EffectcraftApp) -> Option { comp.layer(id).cloned() } -/// Number of effects shown by default: through the last Paint effect (none when there isn't one). -fn default_view(layer: &Layer) -> usize { - layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()).unwrap_or(0) +/// Number of effects shown by default: the Roto Brush's input while a Roto Brush tool is active +/// (or when there is no Paint effect), else through the last Paint effect (none when there +/// isn't one). +fn default_view(layer: &Layer, tool: Tool) -> usize { + let paint = layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()); + match super::roto_tool::roto_index(layer) { + Some(r) if tool.roto_kind(false).is_some() || paint.is_none() => r, + _ => paint.unwrap_or(0), + } } pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { @@ -64,7 +74,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp = app.session.project.comp(cid).cloned().unwrap_or_else(|| effectcraft_engine::project::Comp::new(1, 1, Default::default(), Default::default())); let time = app.session.time(); let fx_count = layer.effects().map(|f| f.groups().count()).unwrap_or(0); - let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer)); + let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer, app.ui.tool)); + let roto = super::roto_tool::roto_index(&layer); // Title strip. let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); @@ -90,9 +101,17 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let tc = super::timecode(&app.session, &comp, time); p.text(pos2(bar.max.x - 10.0, bar.center().y), Align2::RIGHT_CENTER, tc, Tokens::mono(12.0), t.timecode); + // Roto Brush: view / Freeze buttons and the segmentation span bar. + let mut canvas_bottom = bar.min.y; + if roto.is_some() { + super::roto_tool::bar_buttons(app, ui, &p, bar, vr.max.x + 12.0, &layer); + let sb = Rect::from_min_max(pos2(rect.min.x, bar.min.y - 18.0), pos2(rect.max.x, bar.min.y)); + super::roto_tool::span_bar(app, ui, &p, sb, &comp, &layer); + canvas_bottom = sb.min.y; + } // Canvas. - let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, bar.min.y)); + let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, canvas_bottom)); p.rect_filled(area, 0.0, t.pasteboard); let (w, h) = effectcraft_engine::render::source_size(&app.session.project, &layer); let (lw, lh) = if w == 0 { (comp.width as f64, comp.height as f64) } else { (w as f64, h as f64) }; @@ -139,10 +158,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_max(to_screen([tx.rect[0], tx.rect[1]]), to_screen([tx.rect[2], tx.rect[3]])); painter.image(tx.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } + if roto.is_some_and(|r| r == view) { + super::roto_tool::draw_overlay(app, &ctx, &painter, &comp, &layer, scale, &to_screen); + } painter.rect_stroke(frame, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); app.auto.add("layerPanel.canvas", frame, &layer.name); app.auto.add("layerPanel.area", area, "Layer panel"); + // Interaction: Roto Brush / Refine Edge. + if app.ui.tool.roto_kind(false).is_some() { + super::roto_tool::interact(app, ui, &painter, area, &layer, zoom, &to_screen, &to_layer); + return; + } // Interaction: paint tools. let resp = ui.interact(area, egui::Id::new("layer-panel-interact"), Sense::click_and_drag()); let kind = app.ui.tool.paint_kind(); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 4d8e1e247..69d95f664 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -1,10 +1,10 @@ -//! Smaller panels: Preview, Audio, History, Markers, the Home screen and placeholders. +//! Smaller panels: Preview, Audio, History, Markers and placeholders. use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; use serde_json::json; use crate::dock::PanelKind; -use crate::icons::{self, Icon}; +use crate::icons::Icon; use crate::theme::Tokens; use crate::{EffectcraftApp, widgets}; @@ -223,73 +223,3 @@ pub fn markers(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } } - -/// Home screen: new/open, recent, and community links. -pub fn start_screen(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { - let t = app.tokens; - let ctx = ui.ctx().clone(); - let p = ui.painter().with_clip_rect(rect); - p.rect_filled(rect, 0.0, Color32::from_rgb(0x1b, 0x1d, 0x24)); - let c = rect.center(); - let logo = Rect::from_center_size(c + vec2(0.0, -150.0), vec2(64.0, 64.0)); - crate::header::paint_logo(&p, logo); - p.text(c + vec2(0.0, -96.0), Align2::CENTER_CENTER, "EffectCraft", Tokens::semibold(28.0), Color32::WHITE); - p.text(c + vec2(0.0, -66.0), Align2::CENTER_CENTER, "Motion graphics and visual effects, in pure Rust.", Tokens::ui(13.0), t.text_dim); - let mut y = c.y - 30.0; - for (label, id, primary) in [ - ("New Composition", "app.newComp", true), - ("Open Demo Project", "file.openDemoProject", false), - ("Open Project…", "file.open", false), - ("Import Footage…", "file.import", false), - ] { - let r = Rect::from_center_size(pos2(c.x, y + 16.0), vec2(240.0, 32.0)); - if widgets::text_button(ui, r, label, primary, &t, egui::Id::new(("home", id))).clicked() { - let _ = crate::menus::invoke(app, &ctx, id, json!({})); - app.ui.start_screen = false; - } - app.auto.add(&format!("home.{id}"), r, label); - y += 40.0; - } - // Community. - y += 14.0; - let links = [(Icon::Chat, "Join the Discord", "help.discord"), (Icon::Globe, "getartcraft.com", "help.website"), (Icon::Code, "GitHub", "help.github")]; - let w = 170.0; - let mut x = c.x - w * 1.5 - 8.0; - for (icon, label, cmd) in links { - let r = Rect::from_min_size(pos2(x, y), vec2(w, 30.0)); - let resp = ui.interact(r, egui::Id::new(("home-link", cmd)), Sense::click()); - let discord = cmd == "help.discord"; - let bg = if discord { - Color32::from_rgb(0x58, 0x65, 0xf2) - } else if resp.hovered() { - t.hover - } else { - Color32::from_rgb(0x2a, 0x2c, 0x34) - }; - p.rect_filled(r, 15.0, bg); - icons::paint(&p, Rect::from_center_size(pos2(r.min.x + 20.0, r.center().y), vec2(14.0, 14.0)), icon, Color32::WHITE); - p.text(pos2(r.min.x + 34.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::medium(12.0), Color32::WHITE); - app.auto.add(&format!("home.{cmd}"), r, label); - if resp.clicked() { - let _ = app.session.execute(cmd, json!({})); - } - x += w + 8.0; - } - // Siblings. - y += 52.0; - p.text(pos2(c.x, y), Align2::CENTER_CENTER, "More ArtCraft apps", Tokens::ui(11.5), t.text_faint); - y += 18.0; - let sib = effectcraft_engine::links::SIBLINGS; - let sw = 104.0; - let mut x = c.x - sw * sib.len() as f32 / 2.0; - for (name, slug) in sib { - let r = Rect::from_min_size(pos2(x + 2.0, y), vec2(sw - 4.0, 24.0)); - let resp = ui.interact(r, egui::Id::new(("sib", *slug)), Sense::click()); - p.rect_filled(r, 12.0, if resp.hovered() { t.hover } else { Color32::from_rgb(0x24, 0x26, 0x2e) }); - p.text(r.center(), Align2::CENTER_CENTER, *name, Tokens::ui(11.5), t.text); - if resp.clicked() { - let _ = app.session.execute("help.sibling", json!({"app": slug})); - } - x += sw; - } -} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index df60beb3a..2c1f95eea 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; @@ -8,10 +9,12 @@ pub mod effect_controls; pub mod effects_presets; pub mod essential; pub mod expr_bar; +pub mod flowchart; pub mod forms; pub mod fx_widgets; pub mod graph; pub mod graph_tools; +pub mod home; pub mod info; pub mod key_dialogs; pub mod layer_panel; @@ -23,6 +26,8 @@ pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; +pub mod roto_tool; +pub mod script_console; pub mod settings; pub mod shortcut_editor; pub mod text_panels; @@ -67,7 +72,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec match p { PanelKind::Composition => { if app.ui.start_screen { - misc::start_screen(app, ui, rect); + home::show(app, ui, rect); } else { viewer::show(app, ui, rect) } @@ -86,6 +91,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::History => misc::history(app, ui, rect), PanelKind::Markers => misc::markers(app, ui, rect), PanelKind::MaskInterpolation => anim_tools::mask_interpolation(app, ui, rect), + PanelKind::ScriptConsole => script_console::show(app, ui, rect), PanelKind::Wiggler => anim_tools::wiggler(app, ui, rect), PanelKind::Smoother => anim_tools::smoother(app, ui, rect), PanelKind::MotionSketch => anim_tools::motion_sketch(app, ui, rect), @@ -95,7 +101,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), - other => misc::placeholder(app, ui, rect, other), + PanelKind::Flowchart => flowchart::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 6482b555e..545c45d4e 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -548,16 +548,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } dresp.on_hover_text("Project color depth: click to cycle 8/16/32 bpc, Alt-click for Project Settings"); let tr = Rect::from_min_size(pos2(foot.max.x - 30.0, foot.min.y + 3.0), vec2(22.0, 22.0)); - if widgets::icon_button(ui, tr, Icon::Trash, false, &t, egui::Id::new("ptrash")).on_hover_text("Delete selected project items").clicked() { - let sel = app.session.state.project_selection.clone(); - let _ = app.session.edit("Delete Items", None, |proj, st| { - for id in &sel { - proj.items.remove(id); - } - st.project_selection.clear(); - Ok(()) - }); - app.session.sanitize_state(); + if widgets::icon_button(ui, tr, Icon::Trash, false, &t, egui::Id::new("ptrash")).on_hover_text("Delete selected project items").clicked() + && !app.session.state.project_selection.is_empty() + { + actions.push(("project.delete".into(), json!({}))); } app.auto.add("project.delete", tr, "Delete"); for (id, params) in actions { diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index b29bcf185..c02090bc8 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -112,7 +112,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs new file mode 100644 index 000000000..7047f2380 --- /dev/null +++ b/crates/ui-egui/src/panels/roto_tool.rs @@ -0,0 +1,331 @@ +//! Roto Brush and Refine Edge tools in the Layer panel: green foreground strokes, Alt/Option red +//! background strokes, Refine Edge strokes along soft edges; the matte overlay (Alpha Boundary in +//! pink, Alpha, Alpha Overlay), the segmentation span bar and the Freeze button. + +use effectcraft_engine::effects::roto as fx; +use effectcraft_engine::project::{Comp, Layer}; +use effectcraft_engine::render::{EvalCtx, RenderOpts, Renderer}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde_json::json; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// Position of the layer's first Roto Brush & Refine Edge effect. +pub fn roto_index(layer: &Layer) -> Option { + layer.effects()?.groups().position(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == fx::ID)) +} + +fn roto_group(layer: &Layer) -> Option<&effectcraft_engine::project::PropGroup> { + layer.effects()?.groups().nth(roto_index(layer)?) +} + +/// A stroke being drawn. +#[derive(Clone, Debug)] +struct Drag { + layer: u64, + kind: &'static str, + points: Vec<[f64; 2]>, +} + +#[derive(Clone)] +struct Overlay { + key: (u64, u64, i64, u32, String), + tex: egui::TextureHandle, + rect: [f64; 4], +} + +fn c32(c: [f64; 3], a: u8) -> Color32 { + Color32::from_rgba_unmultiplied((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8, a) +} + +/// Draw the matte overlay of the Roto Brush on `layer` (the panel shows the effect's input). +pub fn draw_overlay( + app: &mut EffectcraftApp, + ctx: &egui::Context, + painter: &egui::Painter, + comp: &Comp, + layer: &Layer, + scale: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, +) { + let Some(idx) = roto_index(layer) else { return }; + let o = app.session.state.roto.clone(); + if o.view == "none" { + return; + } + let Some(cid) = app.session.active_comp_id() else { return }; + let time = app.session.time(); + let key = ( + app.session.revision, + layer.id.0, + time.0, + (scale * 1000.0) as u32, + format!("{}{:?}{:?}{}", o.view, o.overlay_color, o.boundary_color, o.overlay_opacity), + ); + let id = egui::Id::new("roto-overlay"); + let cached: Option = ctx.data(|d| d.get_temp(id)); + let ov = match cached { + Some(c) if c.key == key => Some(c), + _ => { + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref(), footage: None }; + let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); + r.expr = app.session.expr.as_deref(); + r.cache = Some(&app.session.layer_cache); + let (Some(inp), Some(out)) = (r.layer_input(&ectx, layer, idx), r.layer_input(&ectx, layer, idx + 1)) else { return }; + let (w, h) = (inp.img.width as usize, inp.img.height as usize); + let a_at = |x: i64, y: i64| -> f32 { + // Output pixel at the same layer point. + let lx = (x as f64 + 0.5 - inp.offset[0]) / inp.scale; + let ly = (y as f64 + 0.5 - inp.offset[1]) / inp.scale; + let (ox, oy) = out.to_px([lx, ly]); + out.img.get(ox.floor() as i64, oy.floor() as i64)[3] + }; + let alpha: Vec = (0..w * h).map(|i| a_at((i % w) as i64, (i / w) as i64)).collect(); + let inside = |x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && alpha[y as usize * w + x as usize] >= 0.5; + let mut px = vec![Color32::TRANSPARENT; w * h]; + let bc = c32(o.boundary_color, 255); + let oc = c32(o.overlay_color, (o.overlay_opacity / 100.0 * 255.0).clamp(0.0, 255.0) as u8); + // View Search Region: the band around the boundary (yellow). + let search = roto_group(layer).filter(|g| g.prop("rotoBrushMatte/viewSearchRegion").is_some_and(|p| p.value.as_bool())).map(|g| { + let r = g.prop("rotoBrushMatte/searchRadius").map(|p| p.value.as_f64()).unwrap_or(15.0) * scale; + let bin: Vec = alpha.iter().map(|a| (*a >= 0.5) as u8).collect(); + let sd = effectcraft_engine::track::roto::matting::signed_distance(&bin, w, h); + (sd, r as f32) + }); + for y in 0..h as i64 { + for x in 0..w as i64 { + let i = y as usize * w + x as usize; + let a = alpha[i]; + px[i] = match o.view.as_str() { + "alpha" => { + let v = (a.clamp(0.0, 1.0) * 255.0) as u8; + Color32::from_rgb(v, v, v) + } + "alphaOverlay" if a < 0.5 => oc, + "alphaOverlay" => Color32::TRANSPARENT, + _ => { + let me = a >= 0.5; + let edge = me && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside(x + dx, y + dy)); + if edge { bc } else { Color32::TRANSPARENT } + } + }; + if let Some((sd, r)) = &search { + let d = sd[i].abs(); + if (d - r).abs() < 0.75 { + px[i] = Color32::from_rgb(255, 220, 0); + } + } + } + } + let img = egui::ColorImage { size: [w, h], pixels: px, source_size: egui::Vec2::new(w as f32, h as f32) }; + let x0 = -inp.offset[0] / inp.scale; + let y0 = -inp.offset[1] / inp.scale; + let rect = [x0, y0, x0 + w as f64 / inp.scale, y0 + h as f64 / inp.scale]; + let tex = ctx.load_texture("roto-overlay", img, egui::TextureOptions::NEAREST); + let c = Overlay { key, tex, rect }; + ctx.data_mut(|d| d.insert_temp(id, c.clone())); + Some(c) + } + }; + if let Some(ov) = ov { + let r = Rect::from_min_max(to_screen([ov.rect[0], ov.rect[1]]), to_screen([ov.rect[2], ov.rect[3]])); + painter.image(ov.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } +} + +/// Pointer handling for the Roto Brush / Refine Edge tools. Returns true when it handled input. +pub fn interact( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + area: Rect, + layer: &Layer, + zoom: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, + to_layer: &dyn Fn(Pos2) -> [f64; 2], +) -> bool { + let ctx = ui.ctx().clone(); + let mods = ui.input(|i| i.modifiers); + let Some(kind) = app.ui.tool.roto_kind(mods.alt) else { return false }; + let resp = ui.interact(area, egui::Id::new("layer-panel-roto"), Sense::click_and_drag()); + let o = app.session.state.roto.clone(); + let refine = kind.starts_with("refine"); + let dia = if refine { o.refine_diameter } else { o.diameter }; + let col = match kind { + "fg" => Color32::from_rgb(40, 220, 60), + "bg" => Color32::from_rgb(235, 50, 40), + "refine" => Color32::from_rgb(110, 200, 255), + _ => Color32::from_rgb(255, 160, 60), + }; + if let Some(hp) = resp.hover_pos() { + let lp = to_layer(hp); + app.pointer_comp = Some([lp[0] as f32, lp[1] as f32]); + ctx.set_cursor_icon(egui::CursorIcon::Crosshair); + let r = (dia * 0.5 * zoom) as f32; + painter.circle_stroke(hp, r, Stroke::new(1.5, col)); + let sign = if kind == "bg" || kind == "refineErase" { "−" } else { "+" }; + painter.text(hp, Align2::CENTER_CENTER, sign, Tokens::semibold(12.0), col); + } + let did = egui::Id::new("layer-panel-roto-drag"); + if resp.drag_started() + && let Some(pos) = ui.input(|i| i.pointer.press_origin()).or(resp.interact_pointer_pos()) + { + ctx.data_mut(|d| d.insert_temp(did, Drag { layer: layer.id.0, kind, points: vec![to_layer(pos)] })); + } + if let Some(mut d) = ctx.data(|d| d.get_temp::(did)) { + if let Some(pos) = resp.interact_pointer_pos() { + let lp = to_layer(pos); + let last = d.points.last().copied().unwrap_or(lp); + if (lp[0] - last[0]).hypot(lp[1] - last[1]) * zoom >= 1.5 { + d.points.push(lp); + ctx.data_mut(|dd| dd.insert_temp(did, d.clone())); + } + } + let width = ((dia * zoom) as f32).max(1.0); + let pts: Vec = d.points.iter().map(|q| to_screen(*q)).collect(); + let c = col.gamma_multiply(0.6); + if pts.len() == 1 { + painter.circle_filled(pts[0], width / 2.0, c); + } else { + painter.add(egui::Shape::line(pts, Stroke::new(width, c))); + } + if resp.drag_stopped() { + ctx.data_mut(|dd| dd.remove::(did)); + run_stroke(app, d.layer, d.kind, &d.points, dia / 2.0); + } + } else if resp.clicked() + && let Some(pos) = resp.interact_pointer_pos() + { + run_stroke(app, layer.id.0, kind, &[to_layer(pos)], dia / 2.0); + } + true +} + +fn run_stroke(app: &mut EffectcraftApp, layer: u64, kind: &str, points: &[[f64; 2]], radius: f64) { + let r = app.session.execute("roto.stroke", json!({"layer": layer, "kind": kind, "points": points, "radius": radius})); + if let Err(e) = r { + app.ui.status = e.to_string(); + } +} + +/// The segmentation span bar (above the panel's bottom bar): the layer's frames, the span, the +/// base frame (yellow), computed frames (green) and stroke frames. Click to go to a frame; drag +/// the span's ends to change it. +pub fn span_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, r: Rect, comp: &Comp, layer: &Layer) { + let t = app.tokens; + let Some(g) = roto_group(layer) else { return }; + let Some(cid) = app.session.active_comp_id() else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let d = fx::data(¶ms); + p.rect_filled(r, 0.0, t.panel_bg); + let lim = effectcraft_engine::roto::frame_limits(comp, layer); + let n = (lim[1] - lim[0] + 1).max(1) as f32; + let track = Rect::from_min_max(pos2(r.min.x + 10.0, r.min.y + 4.0), pos2(r.max.x - 10.0, r.max.y - 4.0)); + let x_of = |f: f64| track.min.x + ((f - lim[0] as f64) / n as f64) as f32 * track.width(); + let f_of = |x: f32| (lim[0] as f64 + ((x - track.min.x) / track.width()).clamp(0.0, 0.9999) as f64 * n as f64).floor() as i64; + p.rect_filled(track, 2.0, t.pasteboard); + if let Some(base) = d.base { + let sp = Rect::from_min_max(pos2(x_of(d.span[0] as f64), track.min.y), pos2(x_of(d.span[1] as f64 + 1.0), track.max.y)); + let frozen = fx::is_frozen(¶ms); + p.rect_filled(sp, 2.0, if frozen { Color32::from_rgb(60, 110, 190) } else { Color32::from_rgb(90, 90, 96) }); + for (f, done) in app.session.roto_computed(cid, layer.id, uid) { + if done { + let cr = Rect::from_min_max(pos2(x_of(f as f64), track.max.y - 4.0), pos2(x_of(f as f64 + 1.0), track.max.y)); + p.rect_filled(cr, 0.0, Color32::from_rgb(70, 200, 90)); + } + } + for f in d.stroke_frames() { + let x = x_of(f as f64 + 0.5); + p.line_segment([pos2(x, track.min.y), pos2(x, track.min.y + 5.0)], Stroke::new(2.0, Color32::WHITE)); + } + let bx = x_of(base as f64 + 0.5); + p.rect_filled(Rect::from_center_size(pos2(bx, track.center().y), vec2(6.0, track.height())), 1.0, Color32::from_rgb(240, 200, 40)); + } + // Current time. + let fps = comp.frame_rate.as_f64(); + let cur = fx::frame_of(layer.layer_time(app.session.time()).seconds(), fps); + let cx = x_of(cur as f64 + 0.5); + p.line_segment([pos2(cx, r.min.y), pos2(cx, r.max.y)], Stroke::new(1.0, t.timecode)); + p.rect_stroke(track, 2.0, Stroke::new(1.0, Color32::from_black_alpha(140)), StrokeKind::Outside); + app.auto.add("layerPanel.rotoSpan", track, "Segmentation span"); + let resp = ui.interact(track, egui::Id::new("roto-span-bar"), Sense::click_and_drag()); + let hid = egui::Id::new("roto-span-handle"); + if resp.drag_started() + && let (Some(pos), Some(_)) = (resp.interact_pointer_pos(), d.base) + { + let (a, b) = (x_of(d.span[0] as f64), x_of(d.span[1] as f64 + 1.0)); + let h = if (pos.x - a).abs() < 6.0 { + 1 + } else if (pos.x - b).abs() < 6.0 { + 2 + } else { + 0 + }; + ui.ctx().data_mut(|m| m.insert_temp(hid, h)); + } + let handle: i32 = ui.ctx().data(|m| m.get_temp(hid)).unwrap_or(0); + if let Some(pos) = resp.interact_pointer_pos() { + let f = f_of(pos.x); + if handle == 0 && (resp.clicked() || resp.dragged()) { + let t = layer.comp_time(comp.frame_rate.tick_of(f)); + let _ = app.session.execute("time.set", json!({"time": t.seconds()})); + } + if handle != 0 && resp.dragged() { + p.line_segment([pos2(pos.x, track.min.y), pos2(pos.x, track.max.y)], Stroke::new(2.0, Color32::from_rgb(240, 200, 40))); + } + if handle != 0 && resp.drag_stopped() { + let k = if handle == 1 { "start" } else { "end" }; + if let Err(e) = app.session.execute("roto.span", json!({"layer": layer.id.0, "effect": uid, k: f})) { + app.ui.status = e.to_string(); + } + } + } + if resp.drag_stopped() { + ui.ctx().data_mut(|m| m.remove::(hid)); + } +} + +/// View-mode toggles and Freeze / Propagate buttons in the Layer panel's bottom bar, from `x`. +pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, bar: Rect, x: f32, layer: &Layer) { + let t = app.tokens; + let mut x = x; + let view = app.session.state.roto.view.clone(); + for (id, label) in [("alpha", "Alpha"), ("alphaBoundary", "Boundary"), ("alphaOverlay", "Overlay")] { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(66.0, 20.0)); + let on = view == id; + if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("roto-view", id))).clicked() { + let v = if on { "none" } else { id }; + let _ = app.session.execute("roto.options", json!({"view": v})); + } + app.auto.add(&format!("layerPanel.roto.view.{id}"), r, label); + x += 70.0; + } + let Some(g) = roto_group(layer) else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let frozen = fx::is_frozen(¶ms); + let running = app.session.is_roto_running(); + x += 8.0; + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(74.0, 20.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, r, label, frozen, &t, egui::Id::new(("roto-freeze", uid))).clicked() { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + if let Err(e) = app.session.execute(id, json!({"layer": layer.id.0, "effect": uid})) { + app.ui.status = e.to_string(); + } + } + app.auto.add("layerPanel.roto.freeze", r, label); + x += 80.0; + if running { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0)); + if widgets::text_button(ui, r, "Stop", false, &t, egui::Id::new("roto-stop")).clicked() { + let _ = app.session.execute("roto.cancel", json!({})); + } + app.auto.add("layerPanel.roto.stop", r, "Stop"); + if let Some(pr) = app.session.roto_progress() { + p.text(pos2(r.max.x + 8.0, bar.center().y), Align2::LEFT_CENTER, pr.banner(), Tokens::ui(11.5), t.text_dim); + } + } +} diff --git a/crates/ui-egui/src/panels/script_console.rs b/crates/ui-egui/src/panels/script_console.rs new file mode 100644 index 000000000..3c258f6ad --- /dev/null +++ b/crates/ui-egui/src/panels/script_console.rs @@ -0,0 +1,144 @@ +//! Window ▸ Script Console: a JavaScript REPL over the After Effects-style scripting object model +//! (`app.project`, comps, layers, properties…). Each run goes through the `script.run` command +//! in the console's persistent context, so variables survive between runs; edits are undoable. + +use egui::{Align2, Rect, Sense, pos2, vec2}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// One line of console output. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ConsoleLine { + pub text: String, + /// `input`, `output`, `result` or `error`. + pub kind: String, +} + +/// Script Console state (serde, so agents can read it). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ScriptConsole { + pub input: String, + pub log: Vec, + /// Previously run inputs (Up/Down recall). + #[serde(default)] + pub history: Vec, +} + +impl ScriptConsole { + fn push(&mut self, kind: &str, text: &str) { + for l in text.lines() { + self.log.push(ConsoleLine { text: l.to_string(), kind: kind.to_string() }); + } + if self.log.len() > 2000 { + let extra = self.log.len() - 2000; + self.log.drain(..extra); + } + } + + /// Record a `script.run` result. + pub fn record(&mut self, code: &str, r: &Result) { + self.push("input", &format!("> {}", code.trim())); + match r { + Ok(v) => { + if let Some(o) = v["output"].as_str().filter(|o| !o.is_empty()) { + self.push("output", o); + } + if let Some(e) = v.get("error").filter(|e| !e.is_null()) { + let line = e["line"].as_u64().map(|l| format!("line {l}: ")).unwrap_or_default(); + self.push("error", &format!("{line}{}", e["message"].as_str().unwrap_or("error"))); + } else if !v["result"].is_null() { + self.push("result", &v["result"].to_string()); + } + } + Err(e) => self.push("error", e), + } + } +} + +/// Run the console input. +pub fn run(app: &mut EffectcraftApp, ctx: &egui::Context) { + let code = app.ui.script_console.input.trim().to_string(); + if code.is_empty() { + return; + } + let r = crate::menus::invoke(app, ctx, "script.run", json!({"code": code, "name": "Script Console", "console": true})); + let con = &mut app.ui.script_console; + con.record(&code, &r); + if con.history.last() != Some(&code) { + con.history.push(code); + } + con.input.clear(); +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let input_h = 86.0; + let bar_h = 26.0; + let out_rect = Rect::from_min_max(rect.min, pos2(rect.max.x, rect.max.y - input_h - bar_h)); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(out_rect, 0.0, t.field_bg); + // Output, newest at the bottom. + let line_h = 16.0; + let visible = ((out_rect.height() - 8.0) / line_h).max(1.0) as usize; + let log = &app.ui.script_console.log; + let start = log.len().saturating_sub(visible); + for (i, l) in log[start..].iter().enumerate() { + let color = match l.kind.as_str() { + "error" => t.danger, + "input" => t.text_faint, + "result" => t.accent, + _ => t.text, + }; + p.text(pos2(out_rect.min.x + 8.0, out_rect.min.y + 4.0 + i as f32 * line_h), Align2::LEFT_TOP, &l.text, Tokens::mono(11.5), color); + } + if log.is_empty() { + p.text( + out_rect.center(), + Align2::CENTER_CENTER, + "JavaScript with the After Effects scripting object model — try app.project.numItems", + Tokens::ui(12.0), + t.text_faint, + ); + } + app.auto.add("scriptConsole.output", out_rect, "Script Console output"); + // Input. + let in_rect = Rect::from_min_size(pos2(rect.min.x + 4.0, out_rect.max.y + 4.0), vec2(rect.width() - 8.0, input_h - 8.0)); + let mut run_now = false; + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(in_rect)); + let te = egui::TextEdit::multiline(&mut app.ui.script_console.input) + .id(egui::Id::new("script-console-input")) + .font(Tokens::mono(12.0)) + .desired_width(in_rect.width()) + .desired_rows(4) + .hint_text("Script… (⌘/Ctrl+Enter runs)"); + let resp = child.add_sized(in_rect.size(), te); + if resp.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && i.modifiers.command) { + run_now = true; + } + app.auto.add("scriptConsole.input", in_rect, "Script Console input"); + // Buttons. + let bar = Rect::from_min_size(pos2(rect.min.x, rect.max.y - bar_h), vec2(rect.width(), bar_h)); + let run_r = Rect::from_min_size(pos2(bar.max.x - 64.0, bar.min.y + 2.0), vec2(56.0, 20.0)); + let clear_r = Rect::from_min_size(pos2(bar.max.x - 128.0, bar.min.y + 2.0), vec2(56.0, 20.0)); + let run_resp = ui.interact(run_r, egui::Id::new("script-console-run"), Sense::click()); + p.rect_filled(run_r, 3.0, if run_resp.hovered() { t.accent_hover } else { t.accent }); + p.text(run_r.center(), Align2::CENTER_CENTER, "Run", Tokens::ui(12.0), egui::Color32::WHITE); + let clear_resp = ui.interact(clear_r, egui::Id::new("script-console-clear"), Sense::click()); + p.rect_stroke(clear_r, 3.0, egui::Stroke::new(1.0, t.field_border), egui::StrokeKind::Inside); + p.text(clear_r.center(), Align2::CENTER_CENTER, "Clear", Tokens::ui(12.0), t.text); + app.auto.add("scriptConsole.run", run_r, "Run"); + app.auto.add("scriptConsole.clear", clear_r, "Clear"); + if run_resp.clicked() { + run_now = true; + } + if clear_resp.clicked() { + app.ui.script_console.log.clear(); + } + if run_now { + let ctx = ui.ctx().clone(); + run(app, &ctx); + } +} diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index b05e523a6..d086486c7 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -93,30 +93,207 @@ const LABEL_W: f32 = 20.0; const NUM_W: f32 = 26.0; const SW: f32 = 19.0; +/// The Timeline's columns in After Effects' order: (id, header label). Right-click a column +/// header to show or hide them; the name column is always shown. +pub const COLUMNS: [(&str, &str); 13] = [ + ("av", "A/V Features"), + ("keys", "Keys"), + ("label", "Label"), + ("num", "#"), + ("name", "Layer Name"), + ("comment", "Comment"), + ("switches", "Switches"), + ("modes", "Modes"), + ("parent", "Parent & Link"), + ("in", "In"), + ("out", "Out"), + ("duration", "Duration"), + ("stretch", "Stretch"), +]; + +/// Is a column shown? +pub fn column_visible(tl: &crate::state::TimelineState, id: &str) -> bool { + match id { + "name" => true, + "modes" => tl.show_modes, + _ => tl.columns.contains(id), + } +} + +/// Show or hide a column (`modes` is the Switches/Modes toggle, F4). +pub fn set_column(tl: &mut crate::state::TimelineState, id: &str, on: bool) -> Result<(), String> { + match id { + "name" if on => Ok(()), + "name" => Err("the name column can't be hidden".into()), + "modes" => { + tl.show_modes = on; + Ok(()) + } + _ if COLUMNS.iter().any(|c| c.0 == id) => { + if on { + tl.columns.insert(id.to_string()); + } else { + tl.columns.remove(id); + } + Ok(()) + } + _ => Err(format!("unknown column `{id}` (one of: {})", COLUMNS.map(|c| c.0).join(", "))), + } +} + +const KEYS_W: f32 = 56.0; +const COMMENT_W: f32 = 120.0; +const TIME_W: f32 = 72.0; +const STRETCH_W: f32 = 60.0; +const PARENT_W: f32 = 116.0; +const MODES_W: f32 = 92.0 + 112.0; + +#[derive(Clone, Copy, Default)] +struct Vis { + av: bool, + keys: bool, + label: bool, + num: bool, + comment: bool, + switches: bool, + modes: bool, + parent: bool, + in_: bool, + out: bool, + duration: bool, + stretch: bool, +} + +impl Vis { + fn of(tl: &crate::state::TimelineState) -> Vis { + let v = |id| column_visible(tl, id); + Vis { + av: v("av"), + keys: v("keys"), + label: v("label"), + num: v("num"), + comment: v("comment"), + switches: v("switches"), + modes: v("modes"), + parent: v("parent"), + in_: v("in"), + out: v("out"), + duration: v("duration"), + stretch: v("stretch"), + } + } + /// Width of every visible column but the name. + fn fixed(&self) -> f32 { + let w = |on: bool, w: f32| if on { w } else { 0.0 }; + w(self.av, AV_W) + + w(self.keys, KEYS_W) + + w(self.label, LABEL_W) + + w(self.num, NUM_W) + + w(self.comment, COMMENT_W) + + w(self.switches, SW * 8.0 + 6.0) + + w(self.modes, MODES_W) + + w(self.parent, PARENT_W) + + w(self.in_, TIME_W) + + w(self.out, TIME_W) + + w(self.duration, TIME_W) + + w(self.stretch, STRETCH_W) + } +} + struct Cols { + vis: Vis, av: f32, + keys: f32, label: f32, num: f32, name: f32, + /// End of the name column. + name_end: f32, + comment: f32, switches: f32, mode: f32, trkmat: f32, parent: f32, + in_: f32, + out: f32, + duration: f32, + stretch: f32, end: f32, } -fn cols(x0: f32, width: f32, show_modes: bool) -> Cols { - let fixed = AV_W + LABEL_W + NUM_W + SW * 8.0 + if show_modes { 92.0 + 112.0 } else { 0.0 } + 116.0; - let name_w = (width - fixed).max(120.0); - let av = x0; - let label = av + AV_W; - let num = label + LABEL_W; - let name = num + NUM_W; - let switches = name + name_w; - let mode = switches + SW * 8.0 + 6.0; - let trkmat = mode + if show_modes { 92.0 } else { 0.0 }; - let parent = trkmat + if show_modes { 112.0 } else { 0.0 }; - Cols { av, label, num, name, switches, mode, trkmat, parent, end: parent + 116.0 } +fn cols(x0: f32, width: f32, vis: Vis) -> Cols { + let name_w = (width - vis.fixed()).max(120.0); + let mut x = x0; + let mut next = |on: bool, w: f32| { + let at = x; + if on { + x += w; + } + at + }; + let av = next(vis.av, AV_W); + let keys = next(vis.keys, KEYS_W); + let label = next(vis.label, LABEL_W); + let num = next(vis.num, NUM_W); + let name = next(true, name_w); + let comment = next(vis.comment, COMMENT_W); + let switches = next(vis.switches, SW * 8.0 + 6.0); + let mode = next(vis.modes, 92.0); + let trkmat = next(vis.modes, 112.0); + let parent = next(vis.parent, PARENT_W); + let in_ = next(vis.in_, TIME_W); + let out = next(vis.out, TIME_W); + let duration = next(vis.duration, TIME_W); + let stretch = next(vis.stretch, STRETCH_W); + let end = next(true, 0.0); + Cols { vis, av, keys, label, num, name, name_end: name + name_w, comment, switches, mode, trkmat, parent, in_, out, duration, stretch, end } +} + +/// Which column a header x position falls in. +fn column_at(cw: &Cols, x: f32) -> Option<&'static str> { + let v = cw.vis; + let spans = [ + ("av", v.av, cw.av, cw.av + AV_W), + ("keys", v.keys, cw.keys, cw.keys + KEYS_W), + ("label", v.label, cw.label, cw.label + LABEL_W), + ("num", v.num, cw.num, cw.num + NUM_W), + ("name", true, cw.name, cw.name_end), + ("comment", v.comment, cw.comment, cw.comment + COMMENT_W), + ("switches", v.switches, cw.switches, cw.switches + SW * 8.0 + 6.0), + ("modes", v.modes, cw.mode, cw.mode + MODES_W), + ("parent", v.parent, cw.parent, cw.parent + PARENT_W), + ("in", v.in_, cw.in_, cw.in_ + TIME_W), + ("out", v.out, cw.out, cw.out + TIME_W), + ("duration", v.duration, cw.duration, cw.duration + TIME_W), + ("stretch", v.stretch, cw.stretch, cw.stretch + STRETCH_W), + ]; + spans.into_iter().find(|(_, on, a, b)| *on && x >= *a && x < *b).map(|s| s.0) +} + +/// Snap a time to the nearest candidate within `tolerance` (all seconds); Shift-dragging the +/// current-time indicator snaps to keyframes, layer in/out points, markers and the work area. +pub fn snap_time(candidates: &[f64], t: f64, tolerance: f64) -> f64 { + candidates.iter().copied().filter(|c| (c - t).abs() <= tolerance).min_by(|a, b| (a - t).abs().total_cmp(&(b - t).abs())).unwrap_or(t) +} + +/// Times the current-time indicator snaps to (comp seconds). +fn snap_candidates(comp: &Comp, rows: &[Row]) -> Vec { + let mut v = vec![0.0, comp.work_area.0.seconds(), comp.work_area.1.seconds(), comp.duration.seconds()]; + v.extend(comp.markers.iter().map(|m| m.time.seconds())); + for l in &comp.layers { + v.push(l.in_point.seconds()); + v.push(l.out_point.seconds()); + v.extend(l.markers.iter().map(|m| l.comp_time(m.time).seconds())); + } + for r in rows { + if let RowKind::Prop { uid } = r.kind + && let Some(l) = comp.layer(r.layer) + && let Some(p) = l.props.find(uid) + { + v.extend(p.keys.iter().map(|k| l.comp_time(k.time).seconds())); + } + } + v } /// Timeline horizontal mapping. @@ -222,15 +399,98 @@ fn reveal_matches(p: &Property, path_matches: &[&str], kind: &str) -> bool { } } +/// The name shown for a layer: its name, or its source's name (Source Name column). +fn display_name(app: &EffectcraftApp, l: &Layer) -> String { + if app.ui.timeline.source_name + && let Some(item) = l.source.item().and_then(|i| app.session.project.item(i)) + { + return item.name.clone(); + } + l.name.clone() +} + +/// The timeline search (After Effects' search field): rows for the properties and groups of a +/// layer whose names contain `q` (lowercase), each with its enclosing groups. A matching group +/// brings its whole contents. +fn search_rows(l: &Layer, q: &str) -> Vec { + fn group_row(l: &Layer, g: &PropGroup, depth: usize, parent: &PropGroup) -> Row { + let fx = matches!(g.kind, GroupKind::Effect { .. }).then_some(g.enabled); + let eye = + (parent.match_id == effectcraft_engine::project::styles::GROUP && g.match_id != effectcraft_engine::project::styles::BLENDING).then_some(g.enabled); + Row { layer: l.id, depth, kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open: true, has_children: !g.children.is_empty(), fx, eye } } + } + fn all(l: &Layer, g: &PropGroup, depth: usize, out: &mut Vec) { + for c in &g.children { + match c { + Node::Group(sg) => { + out.push(group_row(l, sg, depth, g)); + all(l, sg, depth + 1, out); + } + Node::Prop(p) if prop_visible(p, l) => out.push(Row { layer: l.id, depth, kind: RowKind::Prop { uid: p.uid } }), + _ => {} + } + } + } + fn rec(l: &Layer, g: &PropGroup, depth: usize, q: &str, out: &mut Vec) { + for c in &g.children { + match c { + Node::Group(sg) if depth > 1 || group_visible(sg, l) => { + if sg.name.to_lowercase().contains(q) { + out.push(group_row(l, sg, depth, g)); + all(l, sg, depth + 1, out); + continue; + } + let mut inner = vec![]; + rec(l, sg, depth + 1, q, &mut inner); + if !inner.is_empty() { + out.push(group_row(l, sg, depth, g)); + out.extend(inner); + } + } + Node::Prop(p) if prop_visible(p, l) && p.name.to_lowercase().contains(q) => { + out.push(Row { layer: l.id, depth, kind: RowKind::Prop { uid: p.uid } }); + } + _ => {} + } + } + } + let mut out = vec![]; + rec(l, &l.props, 1, q, &mut out); + out +} + +/// The property each reveal key shows (P, S, R, T, A, F, L; `animated` = keyframed or with an +/// expression): (group searched, property match ids). +fn reveal_targets(kind: &str) -> Option<(&'static str, &'static [&'static str])> { + Some(match kind { + "position" => ("transform", &["position", "positionX", "positionY", "positionZ"]), + "scale" => ("transform", &["scale"]), + "rotation" => ("transform", &["rotation", "rotationX", "rotationY", "orientation"]), + "opacity" => ("transform", &["opacity"]), + "anchor" => ("transform", &["anchor"]), + "feather" => ("masks", &["feather"]), + "levels" => ("audio", &["levels"]), + "animated" => ("", &[]), + _ => return None, + }) +} + fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { let tl = &app.ui.timeline; - let search = tl.search.to_lowercase(); + let search = tl.search.trim().to_lowercase(); let mut rows = Vec::new(); for l in &comp.layers { if comp.hide_shy && l.switches.shy { continue; } - if !search.is_empty() && !l.name.to_lowercase().contains(&search) { + if !search.is_empty() { + // Layers whose name or properties match; matching properties are revealed. + let matches = search_rows(l, &search); + if matches.is_empty() && !display_name(app, l).to_lowercase().contains(&search) { + continue; + } + rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); + rows.extend(matches); continue; } rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); @@ -250,112 +510,8 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { if !tl.open_layers.contains(&l.id.0) { continue; } - if let Some(kind) = tl.reveal.first() { - let (group, props): (&str, Vec<&str>) = match kind.as_str() { - "position" => ("transform", vec!["position", "positionX", "positionY", "positionZ"]), - "scale" => ("transform", vec!["scale"]), - "rotation" => ("transform", vec!["rotation", "rotationX", "rotationY", "orientation"]), - "opacity" => ("transform", vec!["opacity"]), - "anchor" => ("transform", vec!["anchor"]), - "feather" => ("masks", vec!["feather"]), - "levels" => ("audio", vec!["levels"]), - _ => ("", vec![]), - }; - match kind.as_str() { - "waveform" => { - if let Some(item) = super::waveform::audio_item(&app.session.project, l) { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); - } - } - "effects" => { - if let Some(fx) = l.effects() { - for g in fx.groups() { - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { - uid: g.uid, - name: g.name.clone(), - open: tl.open_groups.contains(&g.uid), - has_children: true, - fx: Some(g.enabled), - eye: None, - }, - }); - if tl.open_groups.contains(&g.uid) { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - } - } - "masks" => { - if let Some(m) = l.masks() { - for g in m.groups() { - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { - uid: g.uid, - name: g.name.clone(), - open: tl.open_groups.contains(&g.uid), - has_children: true, - fx: None, - eye: None, - }, - }); - if tl.open_groups.contains(&g.uid) { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - } - } - "props" => { - // Animation ▸ Reveal Properties…: the engine picked the uids. - fn groups_in<'a>(g: &'a PropGroup, set: &std::collections::BTreeSet, out: &mut Vec<&'a PropGroup>) { - for sg in g.groups() { - if set.contains(&sg.uid) { - out.push(sg); - } else { - groups_in(sg, set, out); - } - } - } - let mut groups = vec![]; - groups_in(&l.props, &tl.reveal_props, &mut groups); - for g in groups { - let open = tl.open_groups.contains(&g.uid); - rows.push(Row { - layer: l.id, - depth: 1, - kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open, has_children: !g.children.is_empty(), fx: None, eye: None }, - }); - if open { - push_group(&mut rows, l, g, 2, &tl.open_groups); - } - } - let mut found = vec![]; - collect_props(&l.props, &mut found, &|p| prop_visible(p, l) && tl.reveal_props.contains(&p.uid)); - for uid in found { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } }); - } - } - _ => { - let mut found = vec![]; - let root = if group.is_empty() { - Some(&l.props) - } else if group == "masks" { - l.masks() - } else { - l.props.sub(group) - }; - if let Some(g) = root { - collect_props(g, &mut found, &|p| prop_visible(p, l) && reveal_matches(p, &props, kind)); - } - for uid in found { - rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } }); - } - } - } + if !tl.reveal.is_empty() { + reveal_rows(app, l, &mut rows); continue; } for c in &l.props.children { @@ -401,6 +557,95 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { rows } +/// Rows of a twirled-open layer under the reveal shortcuts (one, or several added with Shift): +/// masks, effects, the revealed properties in property-tree order, the Reveal Properties +/// selection, then the waveform. +fn reveal_rows(app: &EffectcraftApp, l: &Layer, rows: &mut Vec) { + let tl = &app.ui.timeline; + let has = |k: &str| tl.reveal.iter().any(|r| r == k); + let group_rows = |rows: &mut Vec, g: &PropGroup, fx: bool| { + for g in g.groups() { + rows.push(Row { + layer: l.id, + depth: 1, + kind: RowKind::Group { + uid: g.uid, + name: g.name.clone(), + open: tl.open_groups.contains(&g.uid), + has_children: true, + fx: fx.then_some(g.enabled), + eye: None, + }, + }); + if tl.open_groups.contains(&g.uid) { + push_group(rows, l, g, 2, &tl.open_groups); + } + } + }; + if has("masks") + && let Some(m) = l.masks() + { + group_rows(rows, m, false); + } + if has("effects") + && let Some(fx) = l.effects() + { + group_rows(rows, fx, true); + } + // P/S/R/T/A/F/L/animated: matching properties, in tree order, each once. + let mut wanted = std::collections::BTreeSet::new(); + for kind in &tl.reveal { + let Some((group, ids)) = reveal_targets(kind) else { continue }; + let root = match group { + "" => Some(&l.props), + "masks" => l.masks(), + g => l.props.sub(g), + }; + if let Some(g) = root { + let mut found = vec![]; + collect_props(g, &mut found, &|p| prop_visible(p, l) && reveal_matches(p, ids, kind)); + wanted.extend(found); + } + } + if !wanted.is_empty() { + let mut found = vec![]; + collect_props(&l.props, &mut found, &|p| wanted.contains(&p.uid)); + rows.extend(found.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + } + if has("props") { + // Animation ▸ Reveal Properties…: the engine picked the uids. + fn groups_in<'a>(g: &'a PropGroup, set: &std::collections::BTreeSet, out: &mut Vec<&'a PropGroup>) { + for sg in g.groups() { + if set.contains(&sg.uid) { + out.push(sg); + } else { + groups_in(sg, set, out); + } + } + } + let mut groups = vec![]; + groups_in(&l.props, &tl.reveal_props, &mut groups); + for g in groups { + let open = tl.open_groups.contains(&g.uid); + rows.push(Row { + layer: l.id, + depth: 1, + kind: RowKind::Group { uid: g.uid, name: g.name.clone(), open, has_children: !g.children.is_empty(), fx: None, eye: None }, + }); + if open { + push_group(rows, l, g, 2, &tl.open_groups); + } + } + let mut found = vec![]; + collect_props(&l.props, &mut found, &|p| prop_visible(p, l) && tl.reveal_props.contains(&p.uid) && !wanted.contains(&p.uid)); + rows.extend(found.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + } + if has("waveform") + && let Some(item) = super::waveform::audio_item(&app.session.project, l) + { + rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); + } +} fn collect_props(g: &PropGroup, out: &mut Vec, f: &dyn Fn(&Property) -> bool) { for c in &g.children { match c { @@ -480,9 +725,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let ruler_h = 34.0; let colhdr_h = 22.0; let footer_h = 24.0; - let fixed = AV_W + LABEL_W + NUM_W + SW * 8.0 + if app.ui.timeline.show_modes { 92.0 + 112.0 } else { 0.0 } + 116.0 + 6.0; - let left_w = (fixed + 190.0).clamp(420.0, (rect.width() * 0.62).max(420.0)); - let cw = cols(rect.min.x, left_w, app.ui.timeline.show_modes); + let vis = Vis::of(&app.ui.timeline); + let left_w = (vis.fixed() + 190.0).clamp(420.0, (rect.width() * 0.72).max(420.0)); + let cw = cols(rect.min.x, left_w, vis); let graph_x0 = rect.min.x + left_w + 1.0; let graph_x1 = rect.max.x - 10.0; ctx.data_mut(|d| d.insert_temp(egui::Id::new("timeline-graph-area"), (graph_x0, graph_x1 - graph_x0))); @@ -500,6 +745,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { p.rect_filled(Rect::from_min_max(pos2(graph_x0, top), rect.max), 0.0, t.tl_bg); p.line_segment([pos2(graph_x0 - 1.0, top), pos2(graph_x0 - 1.0, rect.max.y)], Stroke::new(1.0, t.app_bg)); + let mut actions: Vec<(String, serde_json::Value)> = Vec::new(); // ---- header (left): time display, search, switches. let tc = crate::panels::timecode(&app.session, &comp, time); let tc_rect = Rect::from_min_size(pos2(rect.min.x + 12.0, top + 6.0), vec2(150.0, 24.0)); @@ -616,6 +862,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((a, b)) = run { draw_run(a, b); } + // Disk-cached frames not in RAM (blue), like After Effects. + if app.session.disk_cache.is_some() { + let frames = comp.frame_rate.frame_at(comp.duration); + let opts = app.frame_opts(cid, scale_key as f64 / 1000.0); + let disk = app.frames.disk_frames(&app.render_source(), app.session.revision, cid.0, scale_key, app.view_hash(cid), frames, &opts); + let draw_blue = |a: i64, b: i64| { + let x0 = tm.x(a as f64 * fd); + let x1 = tm.x((b + 1) as f64 * fd); + p.rect_filled(Rect::from_min_max(pos2(x0, cy0), pos2(x1, cy0 + 2.5)), 0.0, t.cache_blue); + }; + let mut run: Option<(i64, i64)> = None; + for f in disk { + run = match run { + Some((a, b)) if f == b + 1 => Some((a, f)), + Some((a, b)) => { + draw_blue(a, b); + Some((f, f)) + } + None => Some((f, f)), + }; + } + if let Some((a, b)) = run { + draw_blue(a, b); + } + } // Ticks + labels. // Label spacing in whole frames (AE: `00:15f`-style seconds:frames labels). let fps_i = fr.as_f64().round().max(1.0) as i64; @@ -649,36 +920,118 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if (rresp.dragged() || rresp.clicked()) && let Some(pt) = rresp.interact_pointer_pos() { - let tt = Tick::from_seconds_f64(tm.t(pt.x).max(0.0)); + let mut secs = tm.t(pt.x).max(0.0); + // Shift-drag: snap to keyframes, in/out points, markers and the work area (8 px). + if ui.input(|i| i.modifiers.shift) { + let cands = snap_candidates(&comp, &build_rows(app, &comp)); + secs = snap_time(&cands, secs, 8.0 / pps.max(1e-6)); + } + let tt = Tick::from_seconds_f64(secs); app.session.set_time(tt); app.stop(); } - // ---- column headers. + // ---- column headers (right-click: show/hide columns; click the name header: Source/Layer Name). let ch = Rect::from_min_max(pos2(rect.min.x, top + header_h), pos2(rect.max.x, top + header_h + colhdr_h)); p.rect_filled(Rect::from_min_max(ch.min, pos2(graph_x0 - 1.0, ch.max.y)), 0.0, t.panel_bg); p.line_segment([pos2(rect.min.x, ch.max.y), pos2(rect.max.x, ch.max.y)], Stroke::new(1.0, t.separator)); let hy = ch.center().y; let hic = |icon: Icon, x: f32| icons::paint(&p, Rect::from_center_size(pos2(x, hy), vec2(12.0, 12.0)), icon, t.text_dim); - hic(Icon::Eye, cw.av + 10.0); - hic(Icon::Speaker, cw.av + 28.0); - hic(Icon::Solo, cw.av + 46.0); - hic(Icon::Lock, cw.av + 64.0); - icons::paint(&p, Rect::from_center_size(pos2(cw.label + 10.0, hy), vec2(10.0, 10.0)), Icon::Keyframe, t.text_dim); - p.text(pos2(cw.num + 6.0, hy), Align2::LEFT_CENTER, "#", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.name + 22.0, hy), Align2::LEFT_CENTER, "Layer Name", Tokens::ui(11.0), t.text_dim); - for (i, icon) in - [Icon::Shy, Icon::Collapse, Icon::Quality, Icon::Fx, Icon::FrameBlend, Icon::MotionBlur, Icon::Adjustment, Icon::Cube].into_iter().enumerate() + let htext = |x: f32, s: &str| p.text(pos2(x, hy), Align2::LEFT_CENTER, s, Tokens::ui(11.0), t.text_dim); + if vis.av { + hic(Icon::Eye, cw.av + 10.0); + hic(Icon::Speaker, cw.av + 28.0); + hic(Icon::Solo, cw.av + 46.0); + hic(Icon::Lock, cw.av + 64.0); + } + if vis.keys { + htext(cw.keys + 6.0, "Keys"); + } + if vis.label { + icons::paint(&p, Rect::from_center_size(pos2(cw.label + 10.0, hy), vec2(10.0, 10.0)), Icon::Keyframe, t.text_dim); + } + if vis.num { + htext(cw.num + 6.0, "#"); + } + let name_hdr = if app.ui.timeline.source_name { "Source Name" } else { "Layer Name" }; + htext(cw.name + 22.0, name_hdr); + if vis.comment { + htext(cw.comment + 6.0, "Comment"); + } + if vis.switches { + for (i, icon) in + [Icon::Shy, Icon::Collapse, Icon::Quality, Icon::Fx, Icon::FrameBlend, Icon::MotionBlur, Icon::Adjustment, Icon::Cube].into_iter().enumerate() + { + hic(icon, cw.switches + SW * i as f32 + SW / 2.0); + } + } + if vis.modes { + htext(cw.mode + 4.0, "Mode"); + htext(cw.mode + 74.0, "T"); + htext(cw.trkmat + 4.0, "Track Matte"); + } + if vis.parent { + icons::paint(&p, Rect::from_center_size(pos2(cw.parent + 10.0, hy), vec2(12.0, 12.0)), Icon::PickWhip, t.text_dim); + htext(cw.parent + 20.0, "Parent & Link"); + } + for (on, x, s) in [(vis.in_, cw.in_, "In"), (vis.out, cw.out, "Out"), (vis.duration, cw.duration, "Duration"), (vis.stretch, cw.stretch, "Stretch")] { + if on { + htext(x + 6.0, s); + } + } + let hdr_rect = Rect::from_min_max(ch.min, pos2(graph_x0 - 1.0, ch.max.y)); + let hresp = ui.interact(hdr_rect, egui::Id::new("tl-colhdr"), Sense::click()); + app.auto.add("timeline.header", hdr_rect, "Columns (right-click)"); + for (id, label) in COLUMNS { + if let Some(x0) = match id { + "av" => vis.av.then_some((cw.av, AV_W)), + "keys" => vis.keys.then_some((cw.keys, KEYS_W)), + "label" => vis.label.then_some((cw.label, LABEL_W)), + "num" => vis.num.then_some((cw.num, NUM_W)), + "name" => Some((cw.name, cw.name_end - cw.name)), + "comment" => vis.comment.then_some((cw.comment, COMMENT_W)), + "switches" => vis.switches.then_some((cw.switches, SW * 8.0 + 6.0)), + "modes" => vis.modes.then_some((cw.mode, MODES_W)), + "parent" => vis.parent.then_some((cw.parent, PARENT_W)), + "in" => vis.in_.then_some((cw.in_, TIME_W)), + "out" => vis.out.then_some((cw.out, TIME_W)), + "duration" => vis.duration.then_some((cw.duration, TIME_W)), + _ => vis.stretch.then_some((cw.stretch, STRETCH_W)), + } { + app.auto.add(&format!("timeline.header.{id}"), Rect::from_min_size(pos2(x0.0, ch.min.y), vec2(x0.1, colhdr_h)), label); + } + } + if hresp.clicked() + && let Some(pt) = hresp.interact_pointer_pos() + && column_at(&cw, pt.x) == Some("name") { - hic(icon, cw.switches + SW * i as f32 + SW / 2.0); + app.ui.timeline.source_name = !app.ui.timeline.source_name; } - if app.ui.timeline.show_modes { - p.text(pos2(cw.mode + 4.0, hy), Align2::LEFT_CENTER, "Mode", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.mode + 74.0, hy), Align2::LEFT_CENTER, "T", Tokens::ui(11.0), t.text_dim); - p.text(pos2(cw.trkmat + 4.0, hy), Align2::LEFT_CENTER, "Track Matte", Tokens::ui(11.0), t.text_dim); + if hresp.secondary_clicked() { + let under = hresp.interact_pointer_pos().and_then(|pt| column_at(&cw, pt.x)); + ctx.data_mut(|d| d.insert_temp(egui::Id::new("tl-colhdr-under"), under.unwrap_or(""))); } - icons::paint(&p, Rect::from_center_size(pos2(cw.parent + 10.0, hy), vec2(12.0, 12.0)), Icon::PickWhip, t.text_dim); - p.text(pos2(cw.parent + 20.0, hy), Align2::LEFT_CENTER, "Parent & Link", Tokens::ui(11.0), t.text_dim); + hresp.context_menu(|ui| { + let under: &str = ctx.data(|d| d.get_temp(egui::Id::new("tl-colhdr-under"))).unwrap_or(""); + if !under.is_empty() && under != "name" && ui.button("Hide This").clicked() { + actions.push(("timeline.column".into(), json!({"column": under, "visible": false}))); + ui.close(); + } + ui.menu_button("Columns", |ui| { + for (id, label) in COLUMNS { + let label = if id == "name" { if app.ui.timeline.source_name { "Source Name" } else { "Layer Name" } } else { label }; + let on = column_visible(&app.ui.timeline, id); + if ui.add_enabled(id != "name", egui::Button::new(label).selected(on)).clicked() { + actions.push(("timeline.column".into(), json!({"column": id, "visible": !on}))); + ui.close(); + } + } + }); + if ui.button(if app.ui.timeline.source_name { "Show Layer Name" } else { "Show Source Name" }).clicked() { + actions.push(("timeline.sourceName".into(), json!({}))); + ui.close(); + } + }); let _ = cw.end; // ---- rows. @@ -706,7 +1059,6 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let mut y = rows_rect.min.y - app.ui.timeline.scroll_y; let selected = app.session.state.selected_layers.clone(); let sel_keys = app.session.state.selected_keys.clone(); - let mut actions: Vec<(String, serde_json::Value)> = Vec::new(); let mut ui_actions: Vec = Vec::new(); let idx_of = |id: LayerId| comp.index_of(id).unwrap_or(0); let graph_on = app.ui.timeline.graph_editor; @@ -756,7 +1108,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ]; for (i, (icon, on, name, applicable)) in av_items.into_iter().enumerate() { let br = Rect::from_center_size(pos2(cw.av + 10.0 + 18.0 * i as f32, cy), vec2(17.0, 17.0)); - if !applicable { + if !applicable || !vis.av { continue; } lp.rect_stroke(br.shrink(1.5), 2.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); @@ -767,19 +1119,24 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } // Label swatch. - let lr = Rect::from_center_size(pos2(cw.label + 10.0, cy), vec2(12.0, 12.0)); - lp.rect_filled(lr, 2.0, t.label(layer.label)); - let lresp = ui.interact(lr, egui::Id::new(("label", layer.id.0)), Sense::click()); - lresp.context_menu(|ui| { - for lab in effectcraft_engine::color::Label::ALL { - let name = if lab == effectcraft_engine::color::Label::None { lab.name().to_string() } else { app.session.prefs.label_name(lab) }; - if ui.button(name).clicked() { - actions.push(("edit.label".into(), json!({"layers": [layer.id.0], "label": lab.name()}))); - ui.close(); + if vis.label { + let lr = Rect::from_center_size(pos2(cw.label + 10.0, cy), vec2(12.0, 12.0)); + lp.rect_filled(lr, 2.0, t.label(layer.label)); + let lresp = ui.interact(lr, egui::Id::new(("label", layer.id.0)), Sense::click()); + lresp.context_menu(|ui| { + for lab in effectcraft_engine::color::Label::ALL { + let name = if lab == effectcraft_engine::color::Label::None { lab.name().to_string() } else { app.session.prefs.label_name(lab) }; + if ui.button(name).clicked() { + actions.push(("edit.label".into(), json!({"layers": [layer.id.0], "label": lab.name()}))); + ui.close(); + } } - } - }); - lp.text(pos2(cw.num + 13.0, cy), Align2::CENTER_CENTER, format!("{}", idx_of(layer.id)), Tokens::ui(11.5), t.text_dim); + }); + } + if vis.num { + lp.text(pos2(cw.num + 13.0, cy), Align2::CENTER_CENTER, format!("{}", idx_of(layer.id)), Tokens::ui(11.5), t.text_dim); + } + layer_cells(app, ui, &lp, &cw, &comp, layer, r, cy, &mut actions); // Twirl + icon + name. let tw = Rect::from_center_size(pos2(cw.name + 8.0, cy), vec2(14.0, 14.0)); let open = app.ui.timeline.open_layers.contains(&layer.id.0); @@ -788,7 +1145,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } app.auto.add(&format!("timeline.layer.{}.twirl", layer.id.0), tw, "twirl"); icons::paint(&lp, Rect::from_center_size(pos2(cw.name + 24.0, cy), vec2(13.0, 13.0)), layer_icon(layer, &app.session.project), t.text_dim); - let name_rect = Rect::from_min_max(pos2(cw.name + 34.0, r.min.y), pos2(cw.switches - 4.0, r.max.y)); + let name_rect = Rect::from_min_max(pos2(cw.name + 34.0, r.min.y), pos2(cw.name_end - 4.0, r.max.y)); let rename_id = egui::Id::new("tl-rename"); let renaming: Option<(u64, String)> = ctx.data(|d| d.get_temp(rename_id)); if let Some((rl, mut buf)) = renaming.filter(|(rl, _)| *rl == layer.id.0) { @@ -806,11 +1163,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } else { let name_col = if is_sel { Color32::WHITE } else { t.text }; let lpn = lp.with_clip_rect(name_rect.intersect(lp.clip_rect())); - lpn.text(pos2(name_rect.min.x, cy), Align2::LEFT_CENTER, &layer.name, Tokens::ui(12.0), name_col); + lpn.text(pos2(name_rect.min.x, cy), Align2::LEFT_CENTER, display_name(app, layer), Tokens::ui(12.0), name_col); } // Row click → select; double-click → rename; drag → reorder (later). + let row_x0 = if vis.num { cw.num } else { cw.name + 16.0 }; let row_resp = - ui.interact(Rect::from_min_max(pos2(cw.num, r.min.y), pos2(cw.switches, r.max.y)), egui::Id::new(("row", layer.id.0)), Sense::click()); + ui.interact(Rect::from_min_max(pos2(row_x0, r.min.y), pos2(cw.name_end, r.max.y)), egui::Id::new(("row", layer.id.0)), Sense::click()); app.auto.add(&format!("timeline.layer.{}.row", layer.id.0), left, &layer.name); if row_resp.clicked() { let m = ui.input(|i| i.modifiers); @@ -846,7 +1204,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { (Icon::Cube, layer.is_3d(), "threeD", layer.source.is_av() || matches!(layer.source, LayerSource::Null)), ]; for (i, (icon, on, name, applicable)) in sws.into_iter().enumerate() { - if !applicable { + if !applicable || !vis.switches { continue; } let br = Rect::from_center_size(pos2(cw.switches + SW * i as f32 + SW / 2.0, cy), vec2(16.0, 16.0)); @@ -917,28 +1275,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } // Parent. - let pw_rect = Rect::from_center_size(pos2(cw.parent + 9.0, cy), vec2(15.0, 15.0)); - let pw = ui.interact(pw_rect, egui::Id::new(("parent-whip", layer.id.0)), Sense::drag()).on_hover_text("Parent pick whip: drag onto a layer"); - icons::paint(&lp, pw_rect.shrink(1.0), Icon::PickWhip, if pw.hovered() || pw.dragged() { t.text } else { t.text_dim }); - app.auto.add(&format!("timeline.layer.{}.pickWhip", layer.id.0), pw_rect, "Parent pick whip"); - if pw.drag_started() { - let pw_state: PickWhip = (0, layer.id.0, 0, pw_rect.center()); - ctx.data_mut(|d| d.insert_temp(pick_whip_id(), pw_state)); - } - let pr_rect = Rect::from_min_size(pos2(cw.parent + 20.0, cy - 9.0), vec2(94.0, 18.0)); - let plabel = layer.parent.and_then(|p| comp.layer(p)).map(|l| format!("{}. {}", idx_of(l.id), l.name)).unwrap_or_else(|| "None".into()); - let presp = widgets::dropdown(ui, pr_rect, &plabel, &t, egui::Id::new(("parent", layer.id.0))); - app.auto.add(&format!("timeline.layer.{}.parent", layer.id.0), pr_rect, "Parent"); - let pop = egui::Id::new(("par-pop", layer.id.0)); - if presp.clicked() { - widgets::open_popup(ui, pop); - } - let cands: Vec<&Layer> = comp.layers.iter().filter(|l| l.id != layer.id).collect(); - let mut popts = vec!["None".to_string(), "-".to_string()]; - popts.extend(cands.iter().map(|l| format!("{}. {}", idx_of(l.id), l.name))); - if let Some(i) = widgets::popup_menu(ui, pop, pr_rect.left_bottom(), &popts, None) { - let par = if i == 0 { serde_json::Value::Null } else { json!(cands[i - 2].id.0) }; - actions.push(("layer.setParent".into(), json!({"layers": [layer.id.0], "parent": par}))); + if vis.parent { + let pw_rect = Rect::from_center_size(pos2(cw.parent + 9.0, cy), vec2(15.0, 15.0)); + let pw = + ui.interact(pw_rect, egui::Id::new(("parent-whip", layer.id.0)), Sense::drag()).on_hover_text("Parent pick whip: drag onto a layer"); + icons::paint(&lp, pw_rect.shrink(1.0), Icon::PickWhip, if pw.hovered() || pw.dragged() { t.text } else { t.text_dim }); + app.auto.add(&format!("timeline.layer.{}.pickWhip", layer.id.0), pw_rect, "Parent pick whip"); + if pw.drag_started() { + let pw_state: PickWhip = (0, layer.id.0, 0, pw_rect.center()); + ctx.data_mut(|d| d.insert_temp(pick_whip_id(), pw_state)); + } + let pr_rect = Rect::from_min_size(pos2(cw.parent + 20.0, cy - 9.0), vec2(94.0, 18.0)); + let plabel = layer.parent.and_then(|p| comp.layer(p)).map(|l| format!("{}. {}", idx_of(l.id), l.name)).unwrap_or_else(|| "None".into()); + let presp = widgets::dropdown(ui, pr_rect, &plabel, &t, egui::Id::new(("parent", layer.id.0))); + app.auto.add(&format!("timeline.layer.{}.parent", layer.id.0), pr_rect, "Parent"); + let pop = egui::Id::new(("par-pop", layer.id.0)); + if presp.clicked() { + widgets::open_popup(ui, pop); + } + let cands: Vec<&Layer> = comp.layers.iter().filter(|l| l.id != layer.id).collect(); + let mut popts = vec!["None".to_string(), "-".to_string()]; + popts.extend(cands.iter().map(|l| format!("{}. {}", idx_of(l.id), l.name))); + if let Some(i) = widgets::popup_menu(ui, pop, pr_rect.left_bottom(), &popts, None) { + let par = if i == 0 { serde_json::Value::Null } else { json!(cands[i - 2].id.0) }; + actions.push(("layer.setParent".into(), json!({"layers": [layer.id.0], "parent": par}))); + } } // Layer bar. ui.set_clip_rect(right_clip); @@ -1237,10 +1598,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, t.tl_bg); let indent = cw.name + 6.0 + 14.0 * row.depth as f32; // Keyframe navigator (in the A/V column, like AE). - if prop.is_animated() { + // (in the Keys column when it is shown). + if prop.is_animated() && (vis.keys || vis.av) { let lt = layer.layer_time(time); let at_key = effectcraft_engine::keyframe::key_at(&prop.keys, lt).is_some(); - let nav_x = cw.av + 18.0; + let nav_x = if vis.keys { cw.keys + 12.0 } else { cw.av + 18.0 }; let prev = Rect::from_center_size(pos2(nav_x, cy), vec2(12.0, 14.0)); let mid = Rect::from_center_size(pos2(nav_x + 16.0, cy), vec2(14.0, 14.0)); let next = Rect::from_center_size(pos2(nav_x + 32.0, cy), vec2(12.0, 14.0)); @@ -1616,6 +1978,101 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// The optional per-layer cells: Comment (double-click to edit), In / Out / Duration (drag to +/// change by frames), Stretch (click: Time Stretch dialog). +#[allow(clippy::too_many_arguments)] +fn layer_cells( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + lp: &egui::Painter, + cw: &Cols, + comp: &Comp, + layer: &Layer, + r: Rect, + cy: f32, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let lid = layer.id.0; + if cw.vis.comment { + let cr = Rect::from_min_max(pos2(cw.comment + 4.0, r.min.y + 1.0), pos2(cw.comment + COMMENT_W - 4.0, r.max.y - 1.0)); + let edit_id = egui::Id::new("tl-comment-edit"); + let editing: Option<(u64, String)> = ctx.data(|d| d.get_temp(edit_id)); + app.auto.add(&format!("timeline.layer.{lid}.comment"), cr, &layer.comment); + if let Some((el, mut buf)) = editing.filter(|(el, _)| *el == lid) { + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(cr)); + let er = child.add(egui::TextEdit::singleline(&mut buf).font(Tokens::ui(11.5)).desired_width(cr.width())); + er.request_focus(); + if er.lost_focus() { + if !ui.input(|i| i.key_pressed(egui::Key::Escape)) && buf != layer.comment { + actions.push(("layer.setComment".into(), json!({"layers": [el], "comment": buf}))); + } + ctx.data_mut(|d| d.remove::<(u64, String)>(edit_id)); + } else { + ctx.data_mut(|d| d.insert_temp(edit_id, (el, buf))); + } + } else { + let resp = ui.interact(cr, egui::Id::new(("tl-comment", lid)), Sense::click()).on_hover_text("Double-click to edit the comment"); + lp.with_clip_rect(cr.intersect(lp.clip_rect())).text(pos2(cr.min.x + 2.0, cy), Align2::LEFT_CENTER, &layer.comment, Tokens::ui(11.5), t.text_dim); + if resp.double_clicked() { + ctx.data_mut(|d| d.insert_temp(edit_id, (lid, layer.comment.clone()))); + } + } + } + let fd = comp.frame_duration().seconds(); + let cells = [ + (cw.vis.in_, cw.in_, "in", layer.in_point.seconds()), + (cw.vis.out, cw.out, "out", layer.out_point.seconds()), + (cw.vis.duration, cw.duration, "duration", (layer.out_point - layer.in_point).seconds()), + ]; + for (on, x, kind, secs) in cells { + if !on { + continue; + } + let cr = Rect::from_min_max(pos2(x + 2.0, r.min.y + 1.0), pos2(x + TIME_W - 2.0, r.max.y - 1.0)); + let tc = crate::panels::timecode(&app.session, comp, Tick::from_seconds_f64(secs) - if kind == "duration" { comp.display_start } else { Tick(0) }); + let resp = ui.interact(cr, egui::Id::new(("tl-cell", kind, lid)), Sense::drag()).on_hover_text("Drag to change"); + lp.text(pos2(cr.min.x + 4.0, cy), Align2::LEFT_CENTER, &tc, Tokens::mono(11.0), if resp.hovered() || resp.dragged() { t.hot_text } else { t.timecode }); + app.auto.add(&format!("timeline.layer.{lid}.{kind}"), cr, &tc); + if resp.hovered() || resp.dragged() { + ctx.set_cursor_icon(egui::CursorIcon::ResizeHorizontal); + } + let acc_id = egui::Id::new(("tl-cell-acc", kind, lid)); + if resp.dragged() { + let mut acc: f32 = ctx.data(|d| d.get_temp(acc_id).unwrap_or(0.0)); + acc += resp.drag_delta().x; + let frames = (acc / 4.0).trunc() as i64; + if frames != 0 { + acc -= frames as f32 * 4.0; + let d = frames as f64 * fd; + let merge = format!("cell-{kind}-{lid}"); + let params = match kind { + "in" => json!({"layers": [lid], "in": (layer.in_point.seconds() + d).max(0.0), "merge": merge}), + _ => json!({"layers": [lid], "out": (layer.out_point.seconds() + d).max(layer.in_point.seconds() + fd), "merge": merge}), + }; + actions.push(("layer.timing".into(), params)); + } + ctx.data_mut(|d| d.insert_temp(acc_id, acc)); + } + if resp.drag_stopped() { + ctx.data_mut(|d| d.remove::(acc_id)); + actions.push(("__endMerge".into(), json!({}))); + } + } + if cw.vis.stretch { + let cr = Rect::from_min_max(pos2(cw.stretch + 2.0, r.min.y + 1.0), pos2(cw.stretch + STRETCH_W - 2.0, r.max.y - 1.0)); + let label = format!("{:.1}%", layer.stretch * 100.0); + let resp = ui.interact(cr, egui::Id::new(("tl-stretch", lid)), Sense::click()).on_hover_text("Time Stretch…"); + lp.text(pos2(cr.min.x + 4.0, cy), Align2::LEFT_CENTER, &label, Tokens::ui(11.5), if resp.hovered() { t.hot_text } else { t.text }); + app.auto.add(&format!("timeline.layer.{lid}.stretch"), cr, &label); + if resp.clicked() { + actions.push(("layer.select".into(), json!({"layers": [lid]}))); + actions.push(("layer.timeStretch".into(), json!({}))); + } + } +} + enum UiAct { ToggleLayer(u64, bool), ToggleGroup(u64), @@ -1916,3 +2373,136 @@ fn value_editor( } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A comp with two solids; "Box" has an animated Position and a Gaussian Blur. + fn app() -> EffectcraftApp { + let mut s = effectcraft_engine::Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 180, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": 320, "height": 180})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 40, "height": 40})).unwrap(); + let comp = s.active_comp().unwrap(); + let bx = comp.layers.iter().find(|l| l.name == "Box").unwrap().id.0; + s.execute("layer.select", json!({"layers": [bx]})).unwrap(); + s.execute("effect.apply", json!({"effect": "ec.blur.gaussian"})).unwrap(); + let pos = s.active_comp().unwrap().layer(LayerId(bx)).unwrap().transform().unwrap().get("position").unwrap().uid; + s.execute("prop.toggleAnimation", json!({"layer": bx, "prop": pos})).unwrap(); + EffectcraftApp::new(s) + } + + fn labels(app: &EffectcraftApp) -> Vec { + let comp = app.session.active_comp().unwrap().clone(); + build_rows(app, &comp) + .into_iter() + .map(|r| { + let l = comp.layer(r.layer).unwrap(); + match r.kind { + RowKind::Layer => l.name.clone(), + RowKind::Group { name, .. } => format!("{}>{name}", " ".repeat(r.depth)), + RowKind::Prop { uid } => format!("{}{}", " ".repeat(r.depth), l.props.find(uid).unwrap().name), + _ => "?".into(), + } + }) + .collect() + } + + #[test] + fn search_filters_layers_and_reveals_matching_properties() { + let mut app = app(); + app.ui.timeline.search = "opac".into(); + let rows = labels(&app); + // Both layers have Transform > Opacity; nothing else shows. + assert_eq!(rows.iter().filter(|r| r.trim() == "Opacity").count(), 2, "{rows:?}"); + assert!(rows.iter().any(|r| r.trim() == ">Transform"), "{rows:?}"); + assert!(!rows.iter().any(|r| r.trim() == "Position"), "{rows:?}"); + // A layer name match shows the layer alone; a group match brings its contents. + app.ui.timeline.search = "plate".into(); + assert_eq!(labels(&app), vec!["Plate".to_string()]); + app.ui.timeline.search = "GAUSSIAN".into(); + let rows = labels(&app); + assert_eq!(rows[0], "Box"); + assert!(rows.iter().any(|r| r.trim() == ">Effects") && rows.iter().any(|r| r.trim() == "Blurriness"), "{rows:?}"); + assert!(!rows.contains(&"Plate".to_string())); + // Nothing matches → no rows. + app.ui.timeline.search = "zzz".into(); + assert!(labels(&app).is_empty()); + // Through the command. + let ctx = egui::Context::default(); + crate::menus::invoke(&mut app, &ctx, "timeline.search", json!({"query": "scale"})).unwrap(); + assert!(labels(&app).iter().any(|r| r.trim() == "Scale")); + } + + #[test] + fn columns_toggle_and_layout() { + let mut app = app(); + let tl = &mut app.ui.timeline; + assert!(column_visible(tl, "switches") && !column_visible(tl, "comment") && !column_visible(tl, "modes")); + set_column(tl, "comment", true).unwrap(); + set_column(tl, "stretch", true).unwrap(); + set_column(tl, "keys", false).unwrap(); + set_column(tl, "label", false).unwrap(); + set_column(tl, "modes", true).unwrap(); + assert!(tl.show_modes && column_visible(tl, "comment") && !column_visible(tl, "label")); + assert!(set_column(tl, "name", false).is_err()); + assert!(set_column(tl, "bogus", true).is_err()); + let vis = Vis::of(tl); + let cw = cols(0.0, 1400.0, vis); + // Hidden columns take no room; visible ones are in After Effects' order. + assert_eq!(cw.keys, cw.label); + assert_eq!(cw.label, cw.num); + assert!(cw.name < cw.comment && cw.comment < cw.switches && cw.switches < cw.mode && cw.parent < cw.stretch); + assert_eq!(column_at(&cw, cw.comment + 5.0), Some("comment")); + assert_eq!(column_at(&cw, cw.name + 5.0), Some("name")); + assert_eq!(column_at(&cw, cw.stretch + 5.0), Some("stretch")); + // Through the command (as agents do). + let ctx = egui::Context::default(); + let r = crate::menus::invoke(&mut app, &ctx, "timeline.column", json!({"column": "duration"})).unwrap(); + assert!(r["columns"].as_array().unwrap().iter().any(|c| c == "duration")); + crate::menus::invoke(&mut app, &ctx, "timeline.column", json!({"column": "duration", "visible": false})).unwrap(); + assert!(!column_visible(&app.ui.timeline, "duration")); + crate::menus::invoke(&mut app, &ctx, "timeline.sourceName", json!({})).unwrap(); + assert!(app.ui.timeline.source_name); + } + + #[test] + fn reveal_keys_add_with_shift_and_uu_reveals_modified() { + let mut app = app(); + let ctx = egui::Context::default(); + crate::menus::invoke(&mut app, &ctx, "timeline.reveal.opacity", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity"]); + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.scale", json!({})).unwrap(); + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.effects", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity", "scale", "effects"]); + let rows = labels(&app); + assert!(rows.iter().any(|r| r.trim() == "Scale") && rows.iter().any(|r| r.trim() == "Opacity"), "{rows:?}"); + assert!(rows.iter().any(|r| r.trim() == ">Gaussian Blur"), "{rows:?}"); + // Tree order: Scale before Opacity. + let si = rows.iter().position(|r| r.trim() == "Scale").unwrap(); + let oi = rows.iter().position(|r| r.trim() == "Opacity").unwrap(); + assert!(si < oi); + // Shift+S again removes Scale. + crate::menus::invoke(&mut app, &ctx, "timeline.revealAdd.scale", json!({})).unwrap(); + assert_eq!(app.ui.timeline.reveal, vec!["opacity", "effects"]); + // U: keyframed properties (Position); UU (twice quickly): every modified property. + crate::menus::invoke(&mut app, &ctx, "anim.reveal", json!({"kind": "keyframes"})).unwrap(); + let n_keys = app.ui.timeline.reveal_props.len(); + assert_eq!(app.ui.timeline.reveal, vec!["props"]); + assert!(n_keys >= 1); + crate::menus::invoke(&mut app, &ctx, "anim.reveal", json!({"kind": "keyframes"})).unwrap(); + assert!(app.ui.timeline.reveal_props.len() > n_keys, "UU reveals modified properties too"); + } + + #[test] + fn cti_snaps_to_the_nearest_candidate() { + assert_eq!(snap_time(&[0.0, 1.0, 2.0], 1.04, 0.1), 1.0); + assert_eq!(snap_time(&[0.0, 1.0, 2.0], 1.5, 0.1), 1.5); + assert_eq!(snap_time(&[1.0, 1.1], 1.06, 0.1), 1.1); + let app = app(); + let comp = app.session.active_comp().unwrap().clone(); + let c = snap_candidates(&comp, &build_rows(&app, &comp)); + assert!(c.contains(&comp.duration.seconds()) && c.contains(&0.0)); + } +} diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b26..364e114e5 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 227e916ad..64babbf22 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1148,6 +1148,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d13501032..2ead3c6ad 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -34,6 +34,7 @@ pub enum Tool { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, PuppetStarch, PuppetBend, @@ -58,10 +59,10 @@ impl Tool { &[Tool::Brush], &[Tool::Clone], &[Tool::Eraser], - &[Tool::RotoBrush], + &[Tool::RotoBrush, Tool::RefineEdge], &[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap], ]; - pub const ALL: [Tool; 29] = [ + pub const ALL: [Tool; 30] = [ Tool::Selection, Tool::Hand, Tool::Zoom, @@ -86,6 +87,7 @@ impl Tool { Tool::Clone, Tool::Eraser, Tool::RotoBrush, + Tool::RefineEdge, Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, @@ -118,6 +120,7 @@ impl Tool { Tool::Clone => "Clone Stamp Tool", Tool::Eraser => "Eraser Tool", Tool::RotoBrush => "Roto Brush Tool", + Tool::RefineEdge => "Refine Edge Tool", Tool::Puppet => "Puppet Position Pin Tool", Tool::PuppetStarch => "Puppet Starch Pin Tool", Tool::PuppetBend => "Puppet Bend Pin Tool", @@ -139,7 +142,7 @@ impl Tool { Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"), Tool::Type | Tool::TypeVertical => Some("Cmd+T"), Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"), - Tool::RotoBrush => Some("Alt+W"), + Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"), Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"), } } @@ -169,6 +172,7 @@ impl Tool { Tool::Clone => Icon::Clone, Tool::Eraser => Icon::Eraser, Tool::RotoBrush => Icon::RotoBrush, + Tool::RefineEdge => Icon::RefineEdge, Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Icon::Puppet, } } @@ -187,6 +191,17 @@ impl Tool { _ => return None, }) } + /// Roto Brush / Refine Edge: the stroke kind they paint (`roto.stroke` kind), without and + /// with Alt/Option. + pub fn roto_kind(self, alt: bool) -> Option<&'static str> { + Some(match (self, alt) { + (Tool::RotoBrush, false) => "fg", + (Tool::RotoBrush, true) => "bg", + (Tool::RefineEdge, false) => "refine", + (Tool::RefineEdge, true) => "refineErase", + _ => return None, + }) + } /// Paint tools, with their stroke kind (`paint.stroke` kind). pub fn paint_kind(self) -> Option<&'static str> { Some(match self { @@ -360,6 +375,18 @@ pub struct TimelineState { /// Properties whose inline expression editor is collapsed. #[serde(default)] pub expr_closed: BTreeSet, + /// Visible optional columns (column header right-click ▸ Columns): `av`, `keys`, `label`, + /// `num`, `comment`, `switches`, `parent`, `in`, `out`, `duration`, `stretch`. The name + /// column is always shown; Modes follows `show_modes` (F4). + #[serde(default = "default_tl_columns")] + pub columns: BTreeSet, + /// The name column shows Source Name instead of Layer Name (click its header). + #[serde(default)] + pub source_name: bool, +} + +pub fn default_tl_columns() -> BTreeSet { + ["av", "label", "num", "switches", "parent"].map(String::from).into_iter().collect() } fn value_graph() -> String { @@ -391,6 +418,8 @@ impl Default for TimelineState { graph_reference: false, graph_transform_box: true, expr_closed: BTreeSet::new(), + columns: default_tl_columns(), + source_name: false, } } } @@ -476,6 +505,9 @@ impl Default for AnimToolsState { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct UiState { pub tool: Tool, + /// Window ▸ Script Console input, output and history. + #[serde(default)] + pub script_console: crate::panels::script_console::ScriptConsole, /// Tool shown in each toolbar slot. pub slot_tools: Vec, pub workspace: String, @@ -570,12 +602,16 @@ pub struct UiState { /// Composition Mini-Flowchart popup position (open when set). #[serde(default)] pub mini_flowchart: Option<[f32; 2]>, + /// Flowchart panel options (layers, effects, solids, direction, root comp). + #[serde(default)] + pub flowchart: crate::panels::flowchart::FlowOptions, } impl Default for UiState { fn default() -> Self { UiState { tool: Tool::Selection, + script_console: Default::default(), slot_tools: Tool::SLOTS.iter().map(|s| s[0]).collect(), workspace: "Default".into(), dock: crate::dock::workspace("Default"), @@ -620,6 +656,7 @@ impl Default for UiState { layer_view: None, anim_tools: AnimToolsState::default(), mini_flowchart: None, + flowchart: Default::default(), } } } diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 5c82e9168..c32ecb248 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -60,6 +60,8 @@ pub struct Tokens { pub cti: Color32, pub work_area: Color32, pub cache_green: Color32, + /// Timeline cache bar for frames held in the disk cache (not in RAM). + pub cache_blue: Color32, pub keyframe: Color32, pub keyframe_selected: Color32, pub pasteboard: Color32, @@ -108,6 +110,7 @@ impl Tokens { cti: Color32::from_rgb(0x2d, 0x8c, 0xeb), work_area: Color32::from_rgb(0x4a, 0x4a, 0x4a), cache_green: Color32::from_rgb(0x3c, 0xa6, 0x4c), + cache_blue: Color32::from_rgb(0x3a, 0x6f, 0xd8), keyframe: Color32::from_rgb(0xa8, 0xa8, 0xa8), keyframe_selected: Color32::from_rgb(0x3d, 0x8f, 0xf5), pasteboard: Color32::from_rgb(0x1a, 0x1a, 0x1a), diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 000000000..45374fba4 --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/crates/ui-egui/tests/ui_script_console.rs b/crates/ui-egui/tests/ui_script_console.rs new file mode 100644 index 000000000..526a52938 --- /dev/null +++ b/crates/ui-egui/tests/ui_script_console.rs @@ -0,0 +1,68 @@ +//! Window ▸ Script Console: the panel shows, runs JavaScript through `script.run` (persistent +//! context) and logs output, results and errors (egui_kittest, UI logic only). + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")).clone(); + let c = egui::pos2(e.rect[0] + e.rect[2] / 2.0, e.rect[1] + e.rect[3] / 2.0); + h.input_mut().events.push(egui::Event::PointerMoved(c)); + for pressed in [true, false] { + h.input_mut().events.push(egui::Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(2); +} + +#[test] +fn script_console_runs_scripts() { + let s = effectcraft_host::session(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::ScriptConsole); + h.run_steps(3); + for id in ["scriptConsole.input", "scriptConsole.output", "scriptConsole.run", "scriptConsole.clear"] { + assert!(h.state().auto.find(id).is_some(), "missing {id}"); + } + h.state_mut().ui.script_console.input = "var c = app.project.items.addComp('Console', 320, 180, 1, 2, 24); writeLn('made'); c.name".into(); + h.run_steps(1); + click(&mut h, "scriptConsole.run"); + assert!(h.state().session.project.items.values().any(|i| i.name == "Console")); + // Variables persist between runs; errors are logged with their line. + h.state_mut().ui.script_console.input = "c.numLayers + 5".into(); + click(&mut h, "scriptConsole.run"); + h.state_mut().ui.script_console.input = "nope()".into(); + click(&mut h, "scriptConsole.run"); + let log: Vec<(String, String)> = h.state().ui.script_console.log.iter().map(|l| (l.kind.clone(), l.text.clone())).collect(); + assert!(log.contains(&("output".into(), "made".into())), "{log:?}"); + assert!(log.contains(&("result".into(), "\"Console\"".into())), "{log:?}"); + assert!(log.contains(&("result".into(), "5".into())), "{log:?}"); + assert!(log.iter().any(|(k, t)| k == "error" && t.contains("line 1") && t.contains("nope")), "{log:?}"); + assert!(h.state_mut().session.undo(), "console edits are undoable"); + click(&mut h, "scriptConsole.clear"); + assert!(h.state().ui.script_console.log.is_empty()); +} + +/// Headless look at the panel (wgpu offscreen); run with +/// `SC_SNAPSHOT=/abs/out.png cargo test -p effectcraft-ui-egui --test ui_script_console -- --ignored`. +#[test] +#[ignore] +fn script_console_snapshot() { + let s = effectcraft_host::session(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::ScriptConsole); + h.run_steps(2); + for code in ["var c = app.project.items.addComp('Console', 320, 180, 1, 2, 24); writeLn('made ' + c.name); c.id", "c.layers.addText('Hi').name", "nope()"] { + h.state_mut().ui.script_console.input = code.into(); + h.run_steps(1); + click(&mut h, "scriptConsole.run"); + } + h.state_mut().ui.script_console.input = "app.project.numItems".into(); + h.run_steps(3); + let img = h.render().expect("render"); + let out = std::env::var("SC_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out/script_console.png").into()); + if let Some(d) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(d).unwrap(); + } + img.save(&out).unwrap(); +} diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 000000000..62144b258 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 000000000..c8f781b87 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 000000000..45db92dde --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 000000000..02cbb0e28 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 000000000..dc8027a6e --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/agents.md b/docs/agents.md index 510bc356d..8689b9dad 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -49,6 +49,7 @@ The server speaks JSON-RPC 2.0 over stdio, one message per line, and supports MC |---|---| | `list_commands {filter?, enabled_only?}` | Discover command ids, their param docs, and whether each can run now. | | `execute_command {command, params?}` | Run any command (undoable). | +| `run_script {code, name?}` | Run JavaScript with the After Effects-style scripting object model (`app.project`, `comp.layers.addText(…)`, `layer.property("ADBE Transform Group").property("ADBE Position").setValueAtTime(…)`…). Returns `{ok, result, output, error: {message, line, column}}`; edits are undoable. | | `get_project` / `get_comp {comp?}` | Project items, comp settings and layers. | | `get_layer {layer, comp?, time?, flat?}` | A layer's property tree. Every node has a `path`. | | `get_property {layer, path, comp?, time?}` | Value at a time, keyframes and expression. | @@ -146,6 +147,56 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + +### Roto Brush & Refine Edge + +The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use +`roto.stroke {layer, kind: fg|bg|refine|refineErase, points: [[x, y], …], frame?, radius?}` +(layer pixels, layer frames). The first foreground stroke applies the Roto Brush & Refine Edge +effect and sets the base frame with a span of 20 frames each side. `roto.propagate {layer, +direction?: forward|backward|both, to?, wait?}` segments the span (in the background unless +`wait`); strokes on any other frame correct it and propagation restarts from there. +`roto.span {start?, end?}`, `roto.freeze {wait?}` / `roto.unfreeze`, `roto.clearStrokes {frame?, +kind?}`, `roto.cancel` and `roto.options {diameter?, refineDiameter?, view?: alphaBoundary|alpha| +alphaOverlay|none}` complete the set; every edit is one undo step. `roto.status {layer, frame?, +matte?, compute?, compareTo?}` reports the base frame, span, computed and stroked frames, frozen +state, job progress and, for a frame, the matte's area, centroid, RLE matte and IoU against a +reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/searchRadius`, +`effects/#1/refineEdgeMatte/decontaminateEdgeColors`, …). + +```sh +effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[360,230]],"radius":10}' \ + roto.stroke '{"layer":"#1","kind":"bg","points":[[40,40],[600,40]],"radius":12}' \ + roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the @@ -191,6 +242,19 @@ effectcraft-cli exec file.exportLottie '{"comp":"Main","path":"main.json","inclu effectcraft-cli exec file.importLottie '{"path":"anim.json"}' main.ecproj --save ``` +Scripts written against After Effects' documented scripting API run with `script`: + +```sh +effectcraft-cli script build.jsx --save-as main.ecproj # empty project unless one is given +effectcraft-cli script --eval 'app.project.item(1).numLayers' main.ecproj --json +effectcraft-cli script tweak.jsx --bridge 9877 # against the live app +``` + +`writeLn`/`$.writeln`/`alert` output is printed, then the value of the last expression. A script +error exits with status 1 and `file:line:col: message`. Scripts may read files only in the +project's folder and may not write files or use the network unless the user turns on Preferences ▸ +Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network. + `file.exportLottie` returns `{path, bytes, warnings}`: the warnings list every feature Lottie cannot express (most effects, cameras and lights, layer styles, audio, video footage…), so an agent can check what a player will not show. A `.lottie` path writes a dotLottie archive. diff --git a/docs/architecture.md b/docs/architecture.md index a030e26c1..7acd39228 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,28 +16,34 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | +| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `model` | 3D models for Advanced 3D: glTF 2.0 (`.gltf`/`.glb`) and OBJ/MTL import (meshes, PBR metallic-roughness materials and textures, node hierarchy, skins, animations), parametric primitives, extruded/bevelled outline meshes and polygon triangulation | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | -| L4 | `host` | A fully wired `Session` (media, expressions, exporter) for the frontends | +| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) whose edits run engine commands; AE match names | +| L4 | `host` | A fully wired `Session` (media, expressions, scripting, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path, track`, -`media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine`. +`media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine, script`, +`script → engine`. ## 2. Time @@ -104,6 +110,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* @@ -223,6 +237,35 @@ lines interpolated); pixel aspect stretches footage, solids and precomps in the (`EvalCtx::par_ratio`, not inherited by children); Invert Alpha and Interpret As Linear Light apply after decoding. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + +**Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`): +strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes +parameter with the base frame and segmentation span. Each frame is segmented by graph cut +(Boykov–Jolly hard constraints, GrabCut colour mixtures, our own Boykov–Kolmogorov max-flow, +coarse to fine) and propagated to the next frame by warping the matte with block optical flow and +re-cutting in a Search Radius band; frames with correction strokes are re-cut with the warped +matte as a soft prior. Refine Edge bands get guided-filter + closed-form matting and +foreground-colour decontamination. Segmentations are derived data: each frame's chain key (Input +Key, settings, strokes from the base frame out) indexes a process-wide cache that renders fill on +demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` keeps the Input +Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes +in the effect. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, @@ -233,6 +276,28 @@ thread and frame; `footage(name)` reads data footage (JSON, CSV, TSV imported wi text kept in the project) through `sourceData`, `sourceText` and `dataValue`. `expr.errors` lists failing expressions for the viewer's error bar. +## 5a. Scripting + +`effectcraft-script` runs JavaScript (boa, a fresh context per run; the Script Console keeps one) +with an object model written from the behaviour the After Effects Scripting Guide documents: +`app`, `Project`, `ItemCollection`, `CompItem`/`FootageItem`/`FolderItem`, `LayerCollection`, +`AVLayer`/`TextLayer`/`ShapeLayer`/`CameraLayer`/`LightLayer`, `PropertyGroup`/`Property` +(values, keyframes, eases, expressions), `TextDocument`, `Shape`, `KeyframeEase`, `MarkerValue`, +`RenderQueue`/`RenderQueueItem`/`OutputModule`, `File`/`Folder` and the enums. The model lives in +`prelude.js`; it reads the session through `__query` (JSON snapshots of items, layers, property +nodes, values and keys) and edits only through `__exec`, which runs engine commands, so every +scripted edit is undoable and identical to the UI's. While a script runs the session is moved +into a thread-local the natives borrow (nothing borrowed enters the JS heap); the comp a call +targets is made active for that command only. `app.beginUndoGroup`/`endUndoGroup` fold the undo +steps in between into one named step. Match names (`ADBE Transform Group`, `ADBE Position`, +`ADBE Gaussian Blur 2`, `ADBE Vector Shape - Rect`…) come from a table in `matchnames.rs`; effect +parameters are `-0001…`. `File` reads are limited to the project's folder, and +writes and the network are refused, unless Preferences ▸ Scripting & Expressions ▸ Allow Scripts +to Write Files and Access Network is on. Entry points: the `script.run` command +(`Session::script`, set by the host), File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`; `.json` +command scripts still run as steps), Window ▸ Script Console, `effectcraft-cli script` and the +MCP `run_script` tool. + ## 6. Commands, the UI seam and automation Every action is a command: an id, a label, a menu path, a shortcut, a parameter description, diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 449a18989..7763ee03d 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -27,6 +27,7 @@ printf '{"id":1,"method":"engine.execute","params":{"command":"comp.new","params |---|---|---| | `engine.execute` | `{command, params?}` | The command's result. Every engine command (`effectcraft-cli commands`) runs exactly as it does headless, so it never opens a dialog: `comp.new {}` creates a default comp. UI-only ids (`tool.*`, `view.*`, `window.*`, `playback.*`, `timeline.*`, `app.*`) fall through to the menu dispatcher. Unknown top-level params are rejected, and the error lists the accepted keys. | | `ui.menu.invoke` | `{id, params?}` | Like a menu click: commands with empty params may open their dialog (New Composition, Solid Settings, file pickers). | +| UI commands (via either) | | `timeline.column {column, visible?}` (av, keys, label, num, comment, switches, modes, parent, in, out, duration, stretch), `timeline.sourceName {value?}`, `timeline.search {query}`, `timeline.revealAdd.`, `flowchart.options {layers?, effects?, solids?, direction?: lr\|tb, comp?}`, `flowchart.graph {comp?}` → `{nodes, edges}`, `window.workspace {name}` (all After Effects workspaces). | | `engine.commands` | `{filter?, enabledOnly?}` | `[{id, label, menu, shortcut, params, enabled, why}]` | | `ui.menu.list` | `{}` | The menu tree with enablement. | | `render.frame` | `{comp?, time?, max_side?, path?, base64?}` | Renders a comp frame through the session, independent of the viewer zoom or resolution. `comp` is an id or name (default: the active comp), `time` is in comp seconds (default: the CTI), `max_side` caps the longest side (0 = full size). Writes a PNG to `path` (default: a temp file) and returns `{comp, time, width, height, path}`. With `base64: true` it returns `{…, png: ""}` and writes no file. | diff --git a/docs/parity.md b/docs/parity.md index 37b2d670a..074c68ba5 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **297 of 298** After Effects 2026 effects by name (99.7%, after M9.11 and Warp Stabilizer; was 257); only the 3D Camera Tracker is left | +| **Effects** | **298 of 298** After Effects 2026 effects by name (100%, after M9.11, Warp Stabilizer and the 3D Camera Tracker; was 257) | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,38 +73,39 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, -the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, -native macOS menus, and wasm threads. (The missing effect categories, 3D Channel, Immersive Video -and OCIO, landed in M9.11.) +What is left is concentrated in large systems: the Warp Stabilizer and mask-tracking UI on top of +the new tracking library, and wasm threads. (The missing effect categories, 3D Channel, Immersive +Video and OCIO, landed in M9.11; the scripting object model in M14.4; Roto Brush and Refine Edge in +M6.7; native macOS menus, the Composition Flowchart, Timeline column/search depth, the Home screen +and every After Effects workspace in the UI-polish wave; PSD/SVG import, WebM/WAV/AIFF output and +the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) ## By area | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | -| Automation | 76% | 3.3 | a JavaScript scripting object model | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | +| Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | -| Compositions | ≈ 75% | 5.0 | marker dialog, flowchart (Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-7) | +| Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | -| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | -| Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | +| 3D | 76% | 10.0 | multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | +| Effects | ≈ 75% | 5.0 | GPU versions of more effects (every After Effects effect exists since M9.11, M12.5 and M12.6) | +| Interface | 70% | 6.0 | Timeline horizontal scrolling of many columns, a richer Learn area (native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | -| Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing -The 3D Camera Tracker (built on the tracking library by other work). Boris FX +None: the 3D Camera Tracker landed in M12.6. Boris FX Mocha and Cineware are third-party and not counted. ### Added in M9.11 (39 effects) @@ -134,10 +135,10 @@ Mocha and Cineware are third-party and not counted. 7. Point tracking and stabilization (in progress). 8. Frame blending, collapse transformations, slip edit. 9. A real 8/16/32-bit pipeline with linear blending and colour management. -10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. +10. ~~Drag-to-dock and floating panels, saved workspaces, native macOS menus.~~ 11. Auto-save, crash recovery, recent projects (in progress). 12. ~~Expression gaps: `sampleImage`, `footage()`~~ (landed with data footage, the error bar and the Expression Language menu; the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 8339afd4d..217e055af 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,14 +23,18 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), + ("opusenc", 0), ("raster", 1), ("keyframe", 1), ("path", 1), + ("psd", 1), ("project", 2), ("text", 2), ("effects", 2), ("track", 2), + ("svg", 2), ("model", 2), ("render", 3), ("media", 3), @@ -40,6 +44,7 @@ const LAYERS: &[(&str, u8)] = &[ ("lottie", 3), ("format", 3), ("engine", 4), + ("script", 4), ("host", 4), ("ui-egui", 5), ("automation", 5), @@ -64,6 +69,8 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("gpu", "render"), ("lottie", "format"), ("host", "engine"), + ("script", "engine"), + ("host", "script"), ("cli", "effectcraft"), ];