diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 4bf3a044d..5bff5cdb0 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -99,7 +99,7 @@ Screenshots of EffectCraft itself, rendered headlessly, showing only procedurall (sidecars `.attribution`). - Bundled extension (ScriptUI panel) with its twelve built-in ease presets: `extensions/scriptui-panels/Ease Presets.jsx` (sidecar `.attribution`). -- Example effect plug-in: `examples/plugins/posterize-bands` (plug-in API v1, WebAssembly). +- Example effect plug-in: `examples/plugins/posterize-bands` (WebAssembly ABI 1). ## Trained models (optional downloads, not bundled) diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 0d78c3619..771d51068 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -1514,7 +1514,7 @@ pub const TIME_DEPENDENT: &[&str] = &[ pub fn is_time_dependent(id: &str) -> bool { // Simulations replay from layer time 0, so every frame differs even when the defaults // happen to look static at the test's two sample times. - id.starts_with("ec.sim.") || TIME_DEPENDENT.contains(&id) + id.starts_with("ec.sim.") || TIME_DEPENDENT.contains(&id) || (!id.starts_with("ec.") && plugin::reads_other_times(id)) } /// Does effect `id` read its own parameters at earlier times (the stepped particle systems sample diff --git a/crates/effects/src/plugin.rs b/crates/effects/src/plugin.rs index 5c973ac57..8bbe17856 100644 --- a/crates/effects/src/plugin.rs +++ b/crates/effects/src/plugin.rs @@ -24,10 +24,12 @@ //! * rendering must be deterministic (same inputs, same output) and may run on several threads //! at once. //! -//! API 2 (native plug-ins; the WebAssembly ABI stays at 1) adds [`EffectPlugin::padding`] and -//! [`EffectPlugin::render_v2`], which gets a [`PluginHost`] to read the effect's input or other -//! layers at other times, and the extra parameter kinds. External effect helpers -//! (`effectcraft-helper-host`) are built on it. +//! API 2 (native plug-ins) adds [`EffectPlugin::padding`] and [`EffectPlugin::render_v2`], which +//! gets a [`PluginHost`] to read the effect's input or other layers at other times, and the extra +//! parameter kinds. External effect helpers (`effectcraft-helper-host`) are built on it. +//! +//! The WebAssembly ABI is versioned separately ([`WASM_API_VERSION`]): ABI 1 is `ec_render`; +//! ABI 2 adds `ec_render2`, which receives the layer's previous and next frames. use std::sync::{Arc, OnceLock, RwLock}; @@ -39,9 +41,12 @@ use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, Params, RenderFn}; /// The plug-in API version this build implements (manifests with API 1 or 2 are accepted). pub const PLUGIN_API_VERSION: u32 = 2; -/// The plug-in API a WebAssembly module implements (its `ec_api_version`). API 2 is for native -/// and helper plug-ins; WebAssembly modules stay at 1. -pub const WASM_API_VERSION: u32 = 1; +/// The newest WebAssembly ABI this build implements (a module's `ec_api_version`). ABI 1 is +/// `ec_render`; ABI 2 adds `ec_render2` and the layer's neighbouring frames. This is separate +/// from the plug-in API version: a module's manifest `api` stays 1 or 2. +pub const WASM_API_VERSION: u32 = 2; +/// The oldest WebAssembly ABI this build still loads. +pub const MIN_WASM_API_VERSION: u32 = 1; /// The most plug-in effects one process can register (commercial effect suites register a few hundred each). pub const MAX_PLUGINS: usize = 1024; @@ -269,6 +274,22 @@ impl PluginImage { fn from_buf(b: Buf) -> PluginImage { PluginImage { width: b.img.width, height: b.img.height, pixels: b.img.data, origin: b.offset, scale: b.scale } } + + /// This image resampled onto a `width × height` buffer grid at `origin`/`scale` (the current + /// frame's); the pixels unchanged when the grids already match. + pub fn on_grid(&self, width: u32, height: u32, origin: [f64; 2], scale: f64) -> Vec<[f32; 4]> { + if self.width == width && self.height == height && self.origin == origin && self.scale == scale { + return self.pixels.clone(); + } + let k = self.scale / scale.max(1e-9); + let src = crate::Image { width: self.width, height: self.height, data: self.pixels.clone() }; + crate::util::gen_image(width, height, |x, y| { + let lx = (x as f64 + 0.5 - origin[0]) * k + self.origin[0]; + let ly = (y as f64 + 0.5 - origin[1]) * k + self.origin[1]; + src.sample_bilinear(lx, ly) + }) + .data + } } /// Services available during an API 2 render ([`EffectPlugin::render_v2`]). @@ -388,6 +409,12 @@ pub trait EffectPlugin: Send + Sync { fn render_v2(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64, _host: &dyn PluginHost) -> Result<(), String> { self.render(frame, params, time) } + /// Whether rendering reads the layer at other times (its neighbouring frames). Default + /// false; the render cache keys a layer with such an effect on its layer time. WebAssembly + /// ABI 2 plug-ins return true. + fn reads_other_times(&self) -> bool { + false + } } static SLOTS: [OnceLock>; MAX_PLUGINS] = [const { OnceLock::new() }; MAX_PLUGINS]; @@ -432,6 +459,12 @@ pub fn plugin(id: &str) -> Option> { SLOTS.iter().filter_map(|s| s.get()).find(|p| p.manifest().id == id).cloned() } +/// Whether the registered plug-in `id` reads the layer at other times (its neighbouring frames), +/// so the render cache must key its layer on the layer time. +pub fn reads_other_times(id: &str) -> bool { + SLOTS.iter().filter_map(|s| s.get()).any(|p| p.manifest().id == id && p.reads_other_times()) +} + fn leak(s: &str) -> &'static str { Box::leak(s.to_string().into_boxed_str()) } diff --git a/crates/engine/src/cmd/effect.rs b/crates/engine/src/cmd/effect.rs index 1d352d192..1ee132955 100644 --- a/crates/engine/src/cmd/effect.rs +++ b/crates/engine/src/cmd/effect.rs @@ -1352,7 +1352,7 @@ pub fn specs() -> Vec { "Load Effect Plug-in...", [], None, - "{path (.wasm / .wat plug-in, API v1) | folder (loads every .wasm in it)}", + "{path (.wasm / .wat plug-in, WebAssembly ABI 1 or 2) | folder (loads every .wasm in it)}", super::always, plugins_load ), diff --git a/crates/engine/src/system/library.rs b/crates/engine/src/system/library.rs index 750ed600a..5b7f63092 100644 --- a/crates/engine/src/system/library.rs +++ b/crates/engine/src/system/library.rs @@ -304,7 +304,8 @@ pub fn parse_catalog(bytes: &[u8]) -> std::result::Result<(Catalog, Vec) fn runs_here(r: &Release) -> bool { // Without a pre-release suffix (`0.18.0-dev` is 0.18.0). let app = env!("CARGO_PKG_VERSION").split(['-', '+']).next().unwrap_or_default(); - r.api == crate::effects::plugin::WASM_API_VERSION && r.min_app.as_deref().is_none_or(|m| !newer(m, app)) + (crate::effects::plugin::MIN_WASM_API_VERSION..=crate::effects::plugin::WASM_API_VERSION).contains(&r.api) + && r.min_app.as_deref().is_none_or(|m| !newer(m, app)) } /// The newest release this build can run (plug-in API and minimum app version). @@ -789,12 +790,13 @@ fn pick(s: &Session, c: &str, p: &Value) -> Result<(String, Package, Release)> { return Err(bad( c, format!( - "{} {} needs plug-in API {} and EffectCraft {} or later; this is EffectCraft {} with WebAssembly plug-in API {}", + "{} {} needs WebAssembly ABI {} and EffectCraft {} or later; this is EffectCraft {} with WebAssembly ABI {} to {}", pkg.name, rel.version, rel.api, rel.min_app.as_deref().unwrap_or("(any)"), env!("CARGO_PKG_VERSION"), + crate::effects::plugin::MIN_WASM_API_VERSION, crate::effects::plugin::WASM_API_VERSION ), )); @@ -1125,8 +1127,9 @@ mod tests { ..Default::default() }; assert_eq!(latest_compatible(&p).map(|r| r.version.as_str()), Some("1.2.0")); - assert!(runs_here(&rel("1.0.0", 1, None)), "API 1 modules still run on a build with a newer plug-in API"); - assert!(!runs_here(&rel("1.0.0", 2, None)), "WebAssembly modules are API 1: an API 2 release can't load"); + assert!(runs_here(&rel("1.0.0", 1, None)), "ABI 1 modules still run on a build with a newer ABI"); + assert!(runs_here(&rel("1.0.0", 2, None)), "ABI 2 modules run on this build"); + assert!(!runs_here(&rel("1.0.0", 3, None)), "a WebAssembly ABI this build doesn't implement can't load"); let j = package_json(&p, None, &Status::default()); assert_eq!(j["needsNewerApp"], true); assert!(latest_compatible(&Package::default()).is_none()); diff --git a/crates/host/tests/it/plugins.rs b/crates/host/tests/it/plugins.rs index 7e4839e6d..33748d366 100644 --- a/crates/host/tests/it/plugins.rs +++ b/crates/host/tests/it/plugins.rs @@ -281,7 +281,7 @@ fn plugin_loading_reports_bad_modules_and_survives_failing_renders() { let files = [ ("a-good.wasm", wat_plugin("org.test.host.good", 1, "i32.const 0").into_bytes()), ("b-garbage.wasm", b"\0asm\x01\0\0\0garbage".to_vec()), - ("c-v2.wasm", wat_plugin("org.test.host.v2", 2, "i32.const 0").into_bytes()), + ("c-v3.wasm", wat_plugin("org.test.host.v3", 3, "i32.const 0").into_bytes()), ("d-empty.wasm", vec![]), ("e-trap.wasm", wat_plugin("org.test.host.trap", 1, "unreachable").into_bytes()), ("notes.txt", b"not a plug-in".to_vec()), @@ -296,7 +296,7 @@ fn plugin_loading_reports_bad_modules_and_survives_failing_renders() { let errs: Vec = r["errors"].as_array().unwrap().iter().map(|e| e["path"].as_str().unwrap().to_string()).collect(); assert_eq!(errs.len(), 3, "{r}"); assert!(errs.iter().all(|p| !p.ends_with(".txt"))); - assert!(r["errors"].to_string().contains("API 2"), "{r}"); + assert!(r["errors"].to_string().contains("ABI 3"), "{r}"); // A single file: errors come back as command errors; loading twice is refused. let good = dir.join("a-good.wasm").to_string_lossy().to_string(); assert!(s.execute("effect.plugins.load", json!({"path": good})).unwrap_err().to_string().contains("already registered")); @@ -311,3 +311,130 @@ fn plugin_loading_reports_bad_modules_and_survives_failing_renders() { assert_eq!(px(&s), before); let _ = std::fs::remove_dir_all(&dir); } + +/// An ABI 2 plug-in (WebAssembly text) that shows the neighbouring frames it got: per pixel it +/// writes |next.red − prev.red| into red, `flags / 8` into green, `prev_time` into blue and 1 +/// into alpha, so a render test can read the frames and their times back from the image. +fn wat_neighbour_plugin(id: &str) -> String { + let manifest = format!(r#"{{"api":2,"id":"{id}","name":"Host {id}","category":"Test Plug-ins","version":"1","params":[]}}"#); + let len = manifest.len(); + let escaped = manifest.replace('"', "\\\""); + format!( + r#"(module + (memory (export "memory") 2) + (data (i32.const 16) "{escaped}") + (func (export "ec_api_version") (result i32) i32.const 2) + (func (export "ec_manifest_ptr") (result i32) i32.const 16) + (func (export "ec_manifest_len") (result i32) i32.const {len}) + (func (export "ec_alloc") (param i32) (result i32) i32.const 1024) + (func (export "ec_render2") + (param $px i32) (param $w i32) (param $h i32) (param $params i32) (param $np i32) + (param $t f64) (param $scale f64) + (param $prev i32) (param $prev_t f64) (param $next i32) (param $next_t f64) (param $flags i32) + (result i32) + (local $i i32) + (local $n i32) + local.get $w + local.get $h + i32.mul + local.set $n + block $done + loop $per_pixel + local.get $i + local.get $n + i32.ge_u + br_if $done + ;; red = |next.red - prev.red| + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $next + local.get $i + i32.const 16 + i32.mul + i32.add + f32.load + local.get $prev + local.get $i + i32.const 16 + i32.mul + i32.add + f32.load + f32.sub + f32.abs + f32.store + ;; green = flags / 8 + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $flags + f32.convert_i32_s + f32.const 8 + f32.div + f32.store offset=4 + ;; blue = prev_time + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $prev_t + f32.demote_f64 + f32.store offset=8 + ;; alpha = 1 + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + f32.const 1 + f32.store offset=12 + local.get $i + i32.const 1 + i32.add + local.set $i + br $per_pixel + end + end + i32.const 0))"# + ) +} + +/// The neighbouring frames are the layer's own pixels at other times: here a precomp whose +/// nested solid fades from 0 to 100 % over one second, at 4 fps. At 0.5 s the neighbours are +/// 0.25 s and 0.75 s, so red (the current frame) was 0.5 and |next − prev| is 0.5. +#[test] +fn abi_v2_plugins_render_with_real_neighbour_frames() { + let mut s = effectcraft_host::session(); + let inner = s.execute("comp.new", json!({"name": "Inner", "width": 40, "height": 20, "frameRate": 4, "duration": 1})).unwrap()["comp"].as_u64().unwrap(); + let solid = s.execute("layer.newSolid", json!({"color": [1.0, 0.0, 0.0], "width": 40, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.addKey", json!({"layer": solid, "path": "transform/opacity", "time": 0.0, "value": 0})).unwrap(); + s.execute("prop.addKey", json!({"layer": solid, "path": "transform/opacity", "time": 1.0, "value": 100})).unwrap(); + let outer = s.execute("comp.new", json!({"name": "Outer", "width": 40, "height": 20, "frameRate": 4, "duration": 1})).unwrap()["comp"].as_u64().unwrap(); + let layer = s.execute("layer.addItem", json!({"comp": outer, "item": inner, "time": 0})).unwrap()["layer"].as_u64().unwrap(); + let outer = effectcraft_engine::project::ItemId(outer); + + let dir = tempfile::TempDir::new().unwrap(); + let wasm = dir.path().join("neighbours.wasm"); + std::fs::write(&wasm, wat_neighbour_plugin("org.test.host.neighbours")).unwrap(); + let r = s.execute("effect.plugins.load", json!({"path": wasm.to_string_lossy()})).unwrap(); + assert_eq!(r["loaded"][0]["id"], "org.test.host.neighbours", "{r}"); + s.execute("effect.apply", json!({"layer": layer, "effect": "org.test.host.neighbours"})).unwrap(); + + let p = s.render(outer, Tick::from_seconds_f64(0.5), Default::default()).get(20, 10); + assert!((p[0] - 0.5).abs() < 0.02, "precomp neighbours: {p:?}"); + // Both neighbours exist, the previous one is the 0.25 s frame, and the frame is opaque. + assert!((p[1] - 3.0 / 8.0).abs() < 0.01, "flags: {p:?}"); + assert!((p[2] - 0.25).abs() < 0.01, "prev time: {p:?}"); + assert_eq!(p[3], 1.0, "{p:?}"); + + // At the very start of the layer only the next frame exists (the previous one is zeroed); + // solids are filled with a quantised colour, hence the 8-bit step. + let p = s.render(outer, Tick::ZERO, Default::default()).get(20, 10); + assert!((p[0] - 0.25).abs() < 0.01, "the next frame's red, with no previous frame: {p:?}"); + assert!((p[1] - 2.0 / 8.0).abs() < 0.01, "only the next frame is valid: {p:?}"); +} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index 4081b61b6..d9b35a91a 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-plugin" -description = "EffectCraft effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1) into the effect registry" +description = "EffectCraft effect plug-ins: loads sandboxed WebAssembly effect modules (WebAssembly ABI v1 and v2) into the effect registry" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index 7c85e144b..da21e22a0 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -1,21 +1,25 @@ //! # effectcraft-plugin (L3) //! -//! Loads **WebAssembly effect plug-ins** (plug-in ABI v1, see `docs/plugins.md`; the native -//! plug-in API 2 additions such as layer parameters and host services are not part of it) into the effect -//! registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or `.wat`) module with -//! no imports that exports: +//! Loads **WebAssembly effect plug-ins** (WebAssembly ABI v1 and v2, see `docs/plugins.md`; the +//! native plug-in API 2 additions such as layer parameters and host services are not part of it) +//! into the effect registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or +//! `.wat`) module with no imports that exports: //! //! | export | signature | | //! |---|---|---| //! | `memory` | memory | linear memory shared with the host | -//! | `ec_api_version` | `() -> i32` | must return 1 | +//! | `ec_api_version` | `() -> i32` | 1 for ABI 1, 2 for ABI 2 (neighbouring frames) | //! | `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | a UTF-8 JSON [`PluginManifest`] in memory | //! | `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | -//! | `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; 0 = ok | +//! | `ec_render` (ABI 1) | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; 0 = ok | +//! | `ec_render2` (ABI 2) | `ec_render`'s arguments, then `prev: i32, prev_time: f64, next: i32, next_time: f64, flags: i32` | the same, with the neighbouring frames; 0 = ok | //! //! For each render the host asks `ec_alloc` for one buffer holding the parameters (`nparams` //! little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s, -//! premultiplied RGBA, row-major), calls `ec_render`, and reads the pixels back. +//! premultiplied RGBA, row-major), calls the render export, and reads the pixels back. An ABI 2 +//! module gets two more pixel buffers after the current frame: the layer's previous and next +//! frames, resampled onto the current grid, and their layer times. The buffers are always +//! allocated (zeroed when a frame does not exist); `flags` bit 0/1 says which are real. //! //! Modules run in the [wasmi](https://github.com/wasmi-labs/wasmi) interpreter (pure Rust, //! MIT/Apache-2.0) with deterministic floats, no imports (no files, network or clock) and a @@ -26,7 +30,9 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] use effectcraft_effects::EffectSpec; -pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, WASM_API_VERSION, register_plugin}; +pub use effectcraft_effects::plugin::{ + EffectPlugin, MIN_WASM_API_VERSION, PLUGIN_API_VERSION, PluginFrame, PluginImage, PluginManifest, PluginParams, WASM_API_VERSION, register_plugin, +}; #[cfg(feature = "wasm")] mod wasm; diff --git a/crates/plugin/src/tests.rs b/crates/plugin/src/tests.rs index f68fde00a..a4fadb26b 100644 --- a/crates/plugin/src/tests.rs +++ b/crates/plugin/src/tests.rs @@ -163,11 +163,15 @@ fn wasm_plugins_are_sandboxed() { // No imports (files, clock, network…). let imports = r#"(module (import "env" "now" (func $now (result f64))) (memory (export "memory") 1))"#; assert!(load_err(load_wasm(imports.as_bytes(), "imports.wat")).contains("import")); - // Garbage, wrong API version, bad manifests. + // Garbage, an ABI this build doesn't implement, bad manifests. A module that claims ABI 2 + // must export `ec_render2` (this one still has only `ec_render`). assert!(load_wasm(b"\0asm nope", "bad.wasm").is_err()); let v2 = gain_module("org.test.v2", "") .replace("(func (export \"ec_api_version\") (result i32) i32.const 1)", "(func (export \"ec_api_version\") (result i32) i32.const 2)"); - assert!(load_err(load_wasm(v2.as_bytes(), "v2.wat")).contains("API 2")); + assert!(load_err(load_wasm(v2.as_bytes(), "v2.wat")).contains("ec_render2")); + let v3 = gain_module("org.test.v3", "") + .replace("(func (export \"ec_api_version\") (result i32) i32.const 1)", "(func (export \"ec_api_version\") (result i32) i32.const 3)"); + assert!(load_err(load_wasm(v3.as_bytes(), "v3.wat")).contains("ABI 3")); let reserved = gain_module("ec.blur.mine", ""); assert!(load_err(load_wasm(reserved.as_bytes(), "reserved.wat")).contains("reserved")); } @@ -194,14 +198,14 @@ fn abi_violations_fail_to_load_with_reasons() { assert!(load_err(load_wasm(m.as_bytes(), "x.wat")).contains("ec_api_version")); let m = edit(&base, "(param $scale f64) (result i32)", "(param $scale f32) (result i32)"); assert!(load_err(load_wasm(m.as_bytes(), "x.wat")).contains("ec_render")); - // API versions other than 1. - for v in ["0", "-1", "2147483647"] { + // WebAssembly ABIs other than 1 and 2. + for v in ["0", "-1", "3", "2147483647"] { let m = edit( &base, "(func (export \"ec_api_version\") (result i32) i32.const 1)", &format!("(func (export \"ec_api_version\") (result i32) i32.const {v})"), ); - assert!(load_err(load_wasm(m.as_bytes(), "x.wat")).contains(&format!("API {v}")), "{v}"); + assert!(load_err(load_wasm(m.as_bytes(), "x.wat")).contains(&format!("ABI {v}")), "{v}"); } // Manifest out of bounds, negative or absurd lengths, not JSON. let ptr = "(func (export \"ec_manifest_ptr\") (result i32) i32.const 16)"; @@ -317,3 +321,168 @@ fn wasm_rendering_is_deterministic() { let twin = load_wasm(gain_module("org.test.det2", body).as_bytes(), "det2.wat").unwrap(); assert_eq!(bits(&render(twin, 0.37, true)), first); } + +/// An ABI 2 module whose `ec_render2` writes what it got from the host into each pixel: red is +/// |next.red − prev.red|, green `flags / 8`, blue `prev_time`, alpha 1. +fn neighbour_module(id: &str) -> String { + let manifest = format!(r#"{{"api":2,"id":"{id}","name":"Neighbours {id}","category":"Test Plug-ins","version":"1","params":[]}}"#); + let len = manifest.len(); + let escaped = manifest.replace('\\', "\\\\").replace('"', "\\\""); + format!( + r#"(module + (memory (export "memory") 2) + (data (i32.const 16) "{escaped}") + (func (export "ec_api_version") (result i32) i32.const 2) + (func (export "ec_manifest_ptr") (result i32) i32.const 16) + (func (export "ec_manifest_len") (result i32) i32.const {len}) + (func (export "ec_alloc") (param i32) (result i32) i32.const 1024) + (func (export "ec_render2") + (param $px i32) (param $w i32) (param $h i32) (param $params i32) (param $np i32) + (param $t f64) (param $scale f64) + (param $prev i32) (param $prev_t f64) (param $next i32) (param $next_t f64) (param $flags i32) + (result i32) + (local $i i32) (local $n i32) + local.get $w + local.get $h + i32.mul + local.set $n + block $done + loop $per_pixel + local.get $i + local.get $n + i32.ge_u + br_if $done + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $next + local.get $i + i32.const 16 + i32.mul + i32.add + f32.load + local.get $prev + local.get $i + i32.const 16 + i32.mul + i32.add + f32.load + f32.sub + f32.abs + f32.store + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $flags + f32.convert_i32_s + f32.const 8 + f32.div + f32.store offset=4 + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + local.get $prev_t + f32.demote_f64 + f32.store offset=8 + local.get $px + local.get $i + i32.const 16 + i32.mul + i32.add + f32.const 1 + f32.store offset=12 + local.get $i + i32.const 1 + i32.add + local.set $i + br $per_pixel + end + end + i32.const 0))"# + ) +} + +fn neighbour_image(red: f32) -> effectcraft_effects::plugin::PluginImage { + effectcraft_effects::plugin::PluginImage { width: 2, height: 1, pixels: vec![[red, 0.0, 0.0, 1.0]; 2], origin: [0.0; 2], scale: 1.0 } +} + +/// A host with frames at two layer times. +struct Host { + time: f64, + fps: f64, + prev: Option<(f64, effectcraft_effects::plugin::PluginImage)>, + next: Option<(f64, effectcraft_effects::plugin::PluginImage)>, +} + +impl effectcraft_effects::plugin::PluginHost for Host { + fn input_at(&self, layer_time: f64) -> Option { + if let Some((t, img)) = &self.prev + && (t - layer_time).abs() < 1e-9 + { + return Some(img.clone()); + } + if let Some((t, img)) = &self.next + && (t - layer_time).abs() < 1e-9 + { + return Some(img.clone()); + } + None + } + + fn layer_input(&self, _param_id: &str, _layer_time: f64) -> Option { + None + } + + fn params_at(&self, _layer_time: f64) -> Option<(Vec, Vec)> { + None + } + + fn frame_rate(&self) -> f64 { + self.fps + } + + fn time(&self) -> f64 { + self.time + } +} + +#[test] +fn abi_v2_receives_neighbour_frames_and_times() { + load_wasm(neighbour_module("org.test.neighbours").as_bytes(), "neighbours.wat").unwrap(); + let p = effectcraft_effects::plugin::plugin("org.test.neighbours").unwrap(); + assert!(effectcraft_effects::plugin::reads_other_times("org.test.neighbours")); + assert!(effectcraft_effects::is_time_dependent("org.test.neighbours")); + // The module has no parameters, so the flattened vector is empty. + let params = effectcraft_effects::plugin::PluginParams { manifest: p.manifest(), flat: &[], texts: &[] }; + let mut px = vec![[0.0f32; 4]; 2]; + let host = Host { time: 0.5, fps: 4.0, prev: Some((0.25, neighbour_image(0.1))), next: Some((0.75, neighbour_image(0.6))) }; + let mut frame = effectcraft_effects::plugin::PluginFrame { width: 2, height: 1, pixels: &mut px, scale: 1.0, origin: [0.0; 2], layer_size: [2.0, 1.0] }; + p.render_v2(&mut frame, ¶ms, 0.5, &host).unwrap(); + assert!((px[0][0] - 0.5).abs() < 1e-6, "|next − prev|: {px:?}"); + assert!((px[0][1] - 3.0 / 8.0).abs() < 1e-6, "both flags: {px:?}"); + assert!((px[0][2] - 0.25).abs() < 1e-6, "previous frame's time: {px:?}"); + assert_eq!(px[0][3], 1.0); +} + +#[test] +fn abi_v2_without_a_host_gets_zeroed_neighbours() { + let spec = load_wasm(neighbour_module("org.test.neighbours.empty").as_bytes(), "empty.wat").unwrap(); + let mut next_id = 1; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next_id), spec.name, [2.0, 1.0]); + let params = effectcraft_effects::flatten_params(&g, &mut |p| p.value.clone()); + let ctx = EffectCtx { params: ¶ms, time: 0.5, layer_size: [2.0, 1.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + let buf = Buf { img: Image::filled(2, 1, [0.5, 0.5, 0.5, 1.0]), offset: [0.0; 2], scale: 1.0 }; + // No host: both neighbour buffers are zeroed and no flag is set. + assert_eq!((spec.render)(&ctx, buf).img.get(0, 0), [0.0, 0.0, 0.0, 1.0]); + // ABI 1 plug-ins keep the time-independent behaviour. + let gain = load_wasm(gain_module("org.test.neighbours.v1", "").as_bytes(), "v1.wat").unwrap(); + assert!(!effectcraft_effects::plugin::reads_other_times("org.test.neighbours.v1")); + assert!(!effectcraft_effects::is_time_dependent("org.test.neighbours.v1")); + assert!(gain.id.starts_with("org.test.")); +} diff --git a/crates/plugin/src/wasm.rs b/crates/plugin/src/wasm.rs index 18a34548f..499d57397 100644 --- a/crates/plugin/src/wasm.rs +++ b/crates/plugin/src/wasm.rs @@ -1,10 +1,11 @@ -//! The wasmi-backed [`EffectPlugin`] (plug-in API v1 ABI, see the crate docs). +//! The wasmi-backed [`EffectPlugin`] (WebAssembly ABI v1 and v2, see the crate docs). use std::sync::Mutex; use wasmi::{Config, Engine, Instance, Linker, Memory, Module, Store, StoreLimits, StoreLimitsBuilder, TypedFunc, WasmParams, WasmResults}; -use crate::{EffectPlugin, PluginFrame, PluginManifest, PluginParams, WASM_API_VERSION}; +use crate::{EffectPlugin, MIN_WASM_API_VERSION, PluginFrame, PluginManifest, PluginParams, WASM_API_VERSION}; +use effectcraft_effects::plugin::{PluginHost, PluginImage}; /// Instruction budget per frame: a fixed allowance plus some per pixel. Generous for per-pixel /// work, but a runaway loop stops instead of hanging the render. @@ -15,32 +16,74 @@ const MAX_MEMORY: usize = 1 << 30; /// The longest manifest read. const MAX_MANIFEST: i32 = 1 << 20; +/// ABI 1: one buffer. type RenderFn = TypedFunc<(i32, i32, i32, i32, i32, f64, f64), i32>; +/// ABI 2: the same call plus the previous and next frame buffers and their layer times. +type Render2Fn = TypedFunc<(i32, i32, i32, i32, i32, f64, f64, i32, f64, i32, f64, i32), i32>; + +enum Renderer { + V1(RenderFn), + V2(Render2Fn), +} /// One instantiated module (renders running on several threads each take their own). struct Inst { + store: Store, + memory: Memory, + alloc: TypedFunc, + render: Renderer, +} + +/// An instance whose ABI version (and so its render export) is not selected yet. +struct Pending { store: Store, instance: Instance, memory: Memory, alloc: TypedFunc, - render: RenderFn, } -impl Inst { +impl Pending { + /// A typed export by name, reporting a missing or mistyped export like [`missing`]. fn func(&self, name: &str) -> Result, String> { self.instance.get_typed_func::(&self.store, name).map_err(|_| missing(name)) } + + /// Select the render export for `abi`. + fn finish(self, abi: u32) -> Result { + let render = match abi { + 1 => Renderer::V1(self.instance.get_typed_func::<_, i32>(&self.store, "ec_render").map_err(|_| missing("ec_render"))?), + 2 => Renderer::V2(self.instance.get_typed_func::<_, i32>(&self.store, "ec_render2").map_err(|_| missing("ec_render2"))?), + v => return Err(unsupported(v as i32)), + }; + Ok(Inst { store: self.store, memory: self.memory, alloc: self.alloc, render }) + } } +/// The error for a missing or mistyped export. fn missing(name: &str) -> String { format!("the module exports no `{name}` (or with the wrong signature)") } +/// The error for an ABI this build does not implement. +fn unsupported(v: i32) -> String { + format!("the module implements WebAssembly ABI {v}, this build implements ABI {MIN_WASM_API_VERSION} to {WASM_API_VERSION}") +} + +/// The module's `ec_api_version`, when this build implements it. +fn abi_of(v: i32) -> Result { + match u32::try_from(v) { + Ok(v) if (MIN_WASM_API_VERSION..=WASM_API_VERSION).contains(&v) => Ok(v), + _ => Err(unsupported(v)), + } +} + +/// The error for a wasmi or trap failure. fn err(e: impl std::fmt::Display) -> String { e.to_string() } -fn instantiate(engine: &Engine, module: &Module) -> Result { +/// Instantiate `module` in a fresh, limited store and read its memory and allocator. +fn begin(engine: &Engine, module: &Module) -> Result { let mut store = Store::new(engine, StoreLimitsBuilder::new().memory_size(MAX_MEMORY).instances(1).build()); store.limiter(|l| l); store.set_fuel(FUEL_BASE).map_err(err)?; @@ -48,14 +91,20 @@ fn instantiate(engine: &Engine, module: &Module) -> Result { let instance = linker.instantiate_and_start(&mut store, module).map_err(|e| format!("instantiation failed: {e}"))?; let memory = instance.get_memory(&store, "memory").ok_or("the module exports no `memory`")?; let alloc = instance.get_typed_func::(&store, "ec_alloc").map_err(|_| missing("ec_alloc"))?; - let render = instance.get_typed_func::<(i32, i32, i32, i32, i32, f64, f64), i32>(&store, "ec_render").map_err(|_| missing("ec_render"))?; - Ok(Inst { store, instance, memory, alloc, render }) + Ok(Pending { store, instance, memory, alloc }) +} + +/// Instantiate `module` and select the render export for `abi`. +fn instantiate(engine: &Engine, module: &Module, abi: u32) -> Result { + begin(engine, module)?.finish(abi) } /// A loaded WebAssembly effect plug-in. pub struct WasmPlugin { manifest: PluginManifest, source: String, + /// The WebAssembly ABI the module implements (`ec_api_version`). + abi: u32, engine: Engine, module: Module, /// Idle instances. @@ -63,6 +112,7 @@ pub struct WasmPlugin { } impl WasmPlugin { + /// Load `bytes` and its manifest; errors on imports, a bad manifest or a missing export. pub fn new(bytes: &[u8], source: &str) -> Result { let mut config = Config::default(); config.consume_fuel(true); @@ -71,51 +121,42 @@ impl WasmPlugin { if let Some(i) = module.imports().next() { return Err(format!("plug-ins can't import anything (the module imports `{}.{}`)", i.module(), i.name())); } - let mut inst = instantiate(&engine, &module)?; - let version = inst.func::<(), i32>("ec_api_version")?; - let v = version.call(&mut inst.store, ()).map_err(err)?; - if u32::try_from(v).ok() != Some(WASM_API_VERSION) { - return Err(format!("the module implements plug-in API {v}, this build implements API {WASM_API_VERSION} for WebAssembly")); - } - let ptr = inst.func::<(), i32>("ec_manifest_ptr")?.call(&mut inst.store, ()).map_err(err)?; - let len = inst.func::<(), i32>("ec_manifest_len")?.call(&mut inst.store, ()).map_err(err)?; + let mut pending = begin(&engine, &module)?; + let version = pending.func::<(), i32>("ec_api_version")?; + let abi = abi_of(version.call(&mut pending.store, ()).map_err(err)?)?; + let ptr = pending.func::<(), i32>("ec_manifest_ptr")?.call(&mut pending.store, ()).map_err(err)?; + let len = pending.func::<(), i32>("ec_manifest_len")?.call(&mut pending.store, ()).map_err(err)?; if !(0..=MAX_MANIFEST).contains(&len) { return Err(format!("bad manifest length {len}")); } let mut text = vec![0u8; len as usize]; - inst.memory.read(&inst.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; + pending.memory.read(&pending.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; let manifest: PluginManifest = serde_json::from_slice(&text).map_err(|e| format!("bad manifest JSON: {e}"))?; manifest.validate()?; if let Some(p) = manifest.params.iter().find(|p| p.kind.needs_v2()) { return Err(format!("parameter `{}`: WebAssembly plug-ins use plug-in API 1 parameter types (no point3, layer, text or hidden)", p.id)); } - Ok(WasmPlugin { manifest, source: source.to_string(), engine, module, pool: Mutex::new(vec![inst]) }) + let inst = pending.finish(abi)?; + Ok(WasmPlugin { manifest, source: source.to_string(), abi, engine, module, pool: Mutex::new(vec![inst]) }) } + /// An idle instance, or a fresh one when the pool is empty. fn take(&self) -> Result { if let Some(i) = self.pool.lock().ok().and_then(|mut p| p.pop()) { return Ok(i); } - instantiate(&self.engine, &self.module) + instantiate(&self.engine, &self.module, self.abi) } + /// Return a working instance to the pool. fn give_back(&self, inst: Inst) { if let Ok(mut p) = self.pool.lock() { p.push(inst); } } -} -impl EffectPlugin for WasmPlugin { - fn manifest(&self) -> &PluginManifest { - &self.manifest - } - - fn source(&self) -> String { - self.source.clone() - } - - fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + /// ABI 1: parameters, then the pixels; `ec_render`. + fn render_v1(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { let n = frame.pixels.len(); let params_bytes = params.flat.len() * 8; let total = params_bytes + n * 16; @@ -130,25 +171,19 @@ impl EffectPlugin for WasmPlugin { for v in params.flat { buf.extend_from_slice(&v.to_le_bytes()); } - for px in frame.pixels.iter() { - for c in px { - buf.extend_from_slice(&c.to_le_bytes()); - } - } + push_pixels(&mut buf, frame.pixels); inst.memory.write(&mut inst.store, base, &buf).map_err(|e| format!("ec_alloc returned a buffer out of bounds: {e}"))?; let px_ptr = (base + params_bytes) as i32; - let r = inst - .render - .call(&mut inst.store, (px_ptr, frame.width as i32, frame.height as i32, base as i32, params.flat.len() as i32, time, frame.scale)) - .map_err(|e| format!("ec_render trapped: {e}")); + let r = match &inst.render { + Renderer::V1(f) => f + .call(&mut inst.store, (px_ptr, frame.width as i32, frame.height as i32, base as i32, params.flat.len() as i32, time, frame.scale)) + .map_err(|e| format!("ec_render trapped: {e}")), + Renderer::V2(_) => return Err("the module implements WebAssembly ABI 1 but exports ec_render2".into()), + }; let r = match r { Ok(0) => { inst.memory.read(&inst.store, base + params_bytes, &mut buf[params_bytes..]).map_err(err)?; - for (px, b) in frame.pixels.iter_mut().zip(buf[params_bytes..].as_chunks::<16>().0.iter()) { - for (c, v) in px.iter_mut().zip(b.as_chunks::<4>().0.iter()) { - *c = f32::from_le_bytes([v[0], v[1], v[2], v[3]]); - } - } + read_pixels(&buf[params_bytes..], frame.pixels); Ok(()) } Ok(code) => Err(format!("ec_render returned error {code}")), @@ -159,4 +194,170 @@ impl EffectPlugin for WasmPlugin { } r } + + /// ABI 2: parameters, the current frame, then the previous and next frames; `ec_render2`. + /// The neighbour buffers are always allocated (zeroed when absent), so a module can read + /// both without a null check; `flags` says which hold real frames. + fn render_v2( + &self, + frame: &mut PluginFrame, + params: &PluginParams, + time: f64, + prev: Option<(f64, &PluginImage)>, + next: Option<(f64, &PluginImage)>, + ) -> Result<(), String> { + let n = frame.pixels.len(); + let params_bytes = params.flat.len() * 8; + let frame_bytes = n * 16; + let total = params_bytes + frame_bytes * 3; + if total > i32::MAX as usize { + return Err("frame too large for a 32-bit plug-in".into()); + } + let prev_px = prev.map(|(t, img)| (t, img.on_grid(frame.width, frame.height, frame.origin, frame.scale))); + let next_px = next.map(|(t, img)| (t, img.on_grid(frame.width, frame.height, frame.origin, frame.scale))); + let prev = prev_px.as_ref().filter(|(_, px)| px.len() == n); + let next = next_px.as_ref().filter(|(_, px)| px.len() == n); + let mut inst = self.take()?; + inst.store.set_fuel(FUEL_BASE + FUEL_PER_PIXEL * (n as u64 * 3)).map_err(err)?; + let base = inst.alloc.call(&mut inst.store, total as i32).map_err(err)? as u32 as usize; + let mut buf: Vec = Vec::with_capacity(total); + for v in params.flat { + buf.extend_from_slice(&v.to_le_bytes()); + } + push_pixels(&mut buf, frame.pixels); + push_neighbour(&mut buf, prev, frame_bytes); + push_neighbour(&mut buf, next, frame_bytes); + inst.memory.write(&mut inst.store, base, &buf).map_err(|e| format!("ec_alloc returned a buffer out of bounds: {e}"))?; + let px_ptr = (base + params_bytes) as i32; + let prev_ptr = (base + params_bytes + frame_bytes) as i32; + let next_ptr = (base + params_bytes + frame_bytes * 2) as i32; + let flags = i32::from(prev.is_some()) | (i32::from(next.is_some()) << 1); + let prev_time = prev.map_or(0.0, |(t, _)| *t); + let next_time = next.map_or(0.0, |(t, _)| *t); + let r = match &inst.render { + Renderer::V2(f) => f + .call( + &mut inst.store, + ( + px_ptr, + frame.width as i32, + frame.height as i32, + base as i32, + params.flat.len() as i32, + time, + frame.scale, + prev_ptr, + prev_time, + next_ptr, + next_time, + flags, + ), + ) + .map_err(|e| format!("ec_render2 trapped: {e}")), + Renderer::V1(_) => return Err("the module implements WebAssembly ABI 2 but exports ec_render".into()), + }; + let r = match r { + Ok(0) => { + let mut out = vec![0u8; frame_bytes]; + inst.memory.read(&inst.store, base + params_bytes, &mut out).map_err(err)?; + read_pixels(&out, frame.pixels); + Ok(()) + } + Ok(code) => Err(format!("ec_render2 returned error {code}")), + Err(e) => Err(e), + }; + if r.is_ok() { + self.give_back(inst); + } + r + } +} + +/// Append pixels to the little-endian buffer the module reads. +fn push_pixels(buf: &mut Vec, px: &[[f32; 4]]) { + for p in px { + for c in p { + buf.extend_from_slice(&c.to_le_bytes()); + } + } +} + +/// The neighbour's pixels, or `frame_bytes` zeroed bytes when there is no frame. +fn push_neighbour(buf: &mut Vec, neighbour: Option<&(f64, Vec<[f32; 4]>)>, frame_bytes: usize) { + match neighbour { + Some((_, px)) => push_pixels(buf, px), + None => buf.resize(buf.len() + frame_bytes, 0), + } +} + +/// Read the little-endian pixels the module wrote back. +fn read_pixels(bytes: &[u8], pixels: &mut [[f32; 4]]) { + for (px, b) in pixels.iter_mut().zip(bytes.as_chunks::<16>().0.iter()) { + for (c, v) in px.iter_mut().zip(b.as_chunks::<4>().0.iter()) { + *c = f32::from_le_bytes([v[0], v[1], v[2], v[3]]); + } + } +} + +/// The layer times of the frames around `time` on a `fps` grid: the frame before and after it +/// at a frame boundary, the two frames that bracket it between frames. +fn neighbour_times(time: f64, fps: f64) -> (f64, f64) { + if !fps.is_finite() || fps <= 0.0 { + return (time - 1.0 / 30.0, time + 1.0 / 30.0); + } + let step = 1.0 / fps; + let at = time * fps; + // Frame times are not exact in floating point: at NTSC rates `time * fps` often lands a few + // ulps above or below an integer. Treat a near-integer as a frame boundary, so the current + // frame is never handed back as its own neighbour. + if (at - at.round()).abs() < 1e-6 { (time - step, time + step) } else { (at.floor() / fps, at.ceil() / fps) } +} + +impl EffectPlugin for WasmPlugin { + fn manifest(&self) -> &PluginManifest { + &self.manifest + } + + fn source(&self) -> String { + self.source.clone() + } + + fn reads_other_times(&self) -> bool { + self.abi >= 2 + } + + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + if self.abi == 1 { self.render_v1(frame, params, time) } else { self.render_v2(frame, params, time, None, None) } + } + + fn render_v2(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64, host: &dyn PluginHost) -> Result<(), String> { + if self.abi == 1 { + return self.render_v1(frame, params, time); + } + let (prev_time, next_time) = neighbour_times(time, host.frame_rate()); + let prev = host.input_at(prev_time); + let next = host.input_at(next_time); + let prev = prev.as_ref().map(|img| (prev_time, img)); + let next = next.as_ref().map(|img| (next_time, img)); + self.render_v2(frame, params, time, prev, next) + } +} + +#[cfg(test)] +mod tests { + use super::neighbour_times; + + /// NTSC frame times do not multiply back exactly; every frame boundary must still get the + /// frames either side of it, never the frame itself. + #[test] + fn frame_boundaries_get_their_neighbours_at_ntsc_rates() { + let fps = 24000.0 / 1001.0; + for n in 0..2400 { + // 24000/1001 fps at 254 016 000 000 ticks/s, as `Tick` stores it. + let time = (n * 10_594_584_000_i64) as f64 / 254_016_000_000.0; + let (prev, next) = neighbour_times(time, fps); + assert!((prev - (time - 1.0 / fps)).abs() < 1e-9, "frame {n}: prev {prev}, current {time}"); + assert!((next - (time + 1.0 / fps)).abs() < 1e-9, "frame {n}: next {next}, current {time}"); + } + } } diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8a3d7b8ae..0756c47f2 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -689,7 +689,9 @@ fn reads_other_times(layer: &Layer, effects: Option) -> bool { fx.groups().take(effects.unwrap_or(usize::MAX)).any(|g| { g.enabled && matches!(&g.kind, effectcraft_project::GroupKind::Effect { effect } - if effect.starts_with("ec.time.") || effectcraft_effects::reads_param_history(effect)) + if effect.starts_with("ec.time.") + || effectcraft_effects::reads_param_history(effect) + || (!effect.starts_with("ec.") && effectcraft_effects::plugin::reads_other_times(effect))) }) }) } diff --git a/crates/render/src/tests/time.rs b/crates/render/src/tests/time.rs index 715c37ef0..f959401bb 100644 --- a/crates/render/src/tests/time.rs +++ b/crates/render/src/tests/time.rs @@ -1,7 +1,10 @@ //! Time effects through the renderer: neighbouring frames via `EffectHost::self_at`, and the //! layer cache staying correct while scrubbing. +use std::sync::Arc; + use effectcraft_color::Label; +use effectcraft_effects::plugin::{EffectPlugin, PluginFrame, PluginManifest, PluginParams, register_plugin}; use effectcraft_keyframe::{Keyframe, ShapePath, Value}; use effectcraft_project::build::{self, Ids}; use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, MaskMode, Project, Solid}; @@ -157,3 +160,89 @@ fn input_keys_ignore_later_effects() { set(&mut p, "effects/#1/blurriness", 8.0); assert_ne!(key(&p).0, one); } + +/// A plug-in that reads the layer's neighbouring frames (WebAssembly ABI 2) makes its layer's +/// cache key depend on the layer time, even when the source and every property are static. A +/// plug-in that does not keeps the old key (#1977: a static layer stays cached across a scrub). +#[test] +fn abi2_plugin_layers_key_on_layer_time() { + struct NeighbourReader { + m: PluginManifest, + } + + impl EffectPlugin for NeighbourReader { + fn manifest(&self) -> &PluginManifest { + &self.m + } + + fn render(&self, _frame: &mut PluginFrame, _params: &PluginParams, _time: f64) -> Result<(), String> { + Ok(()) + } + + fn reads_other_times(&self) -> bool { + true + } + } + + struct Plain { + m: PluginManifest, + } + + impl EffectPlugin for Plain { + fn manifest(&self) -> &PluginManifest { + &self.m + } + + fn render(&self, _frame: &mut PluginFrame, _params: &PluginParams, _time: f64) -> Result<(), String> { + Ok(()) + } + } + + /// A static white solid with one plug-in effect, registered by `plugin`. + fn plugin_scene(plugin: Arc) -> (Project, ItemId) { + let mut p = Project::default(); + let comp = Comp::new(200, 100, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let sid = p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 200, height: 100, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Solid", LayerSource::Solid { item: sid }, (200, 100), None); + let spec = register_plugin(plugin).unwrap(); + let mut next = p.next_id; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [200.0, 100.0]); + p.next_id = next; + l.props.sub_mut("effects").unwrap().children.push(g.into()); + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) + } + + fn key(p: &Project, cid: ItemId, t: f64) -> Option { + let comp = p.comp(cid).unwrap(); + let ctx = crate::EvalCtx { + project: p, + comp_id: cid, + comp, + time: Tick::from_seconds_f64(t), + expr: None, + footage: None, + property_overrides: &[], + media_overrides: &[], + reads: None, + }; + crate::cache::layer_key(&ctx, &comp.layers[0], 1.0, false, false) + } + + let manifest = |id: &str| PluginManifest { + api: 2, + id: id.into(), + name: "Key".into(), + category: "Test Plug-ins".into(), + version: "1".into(), + author: String::new(), + description: String::new(), + params: vec![], + }; + + let (plain, c1) = plugin_scene(Arc::new(Plain { m: manifest("org.test.render.time-key.plain") })); + assert_eq!(key(&plain, c1, 0.25), key(&plain, c1, 0.5), "a plain plug-in keeps the old key"); + let (reader, c2) = plugin_scene(Arc::new(NeighbourReader { m: manifest("org.test.render.time-key.reader") })); + assert_ne!(key(&reader, c2, 0.25), key(&reader, c2, 0.5), "an ABI 2 plug-in keys on the layer time"); +} diff --git a/docs/architecture.md b/docs/architecture.md index a04907aec..1e91da215 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,7 +43,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `interchange` | Premiere Pro interop through timeline interchange (FCP7 XML, FCPXML, OTIO, EDL, AAF, OMF via FilmCraft's `filmcraft-interchange`): sequences ↔ compositions (§6a) | | L3 | `aep` | After Effects projects in one crate (like `lottie`): `container` reads and writes the `.aep` container (RIFX `Egg!`) as a lossless chunk tree with a byte-identical round trip, bounded for hostile input, reads the `.aepx` XML form into the same tree, and has a typed view of items, layers, properties, effects, masks, shapes, text, markers, footage, cameras, lights and layer styles, from our own observations of generated files ([aep-format.md](aep-format.md)); `import` maps the typed view into a project (items, settings, compositions, every layer kind, Transform, keyframes, expressions, masks, text, shapes, cameras, lights, mattes, markers, time remapping, layer styles, the effects of a shared table, footage) with a report of everything not imported ([aep-import.md](aep-import.md)); `export` builds every chunk in code (prototypes in the layout After Effects writes, cloned and edited only at verified fields) for the same scope, with a report of everything not exported, verified by opening the output in After Effects ([aep-export.md](aep-export.md)) | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | -| L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | +| L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (WebAssembly ABI v1 and v2, the latter with neighbouring frames; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | | L3 | `helper-host` | External effect helpers: finds helper programs, runs each as a child process and registers its effects (plug-in API 2) over a versioned pipe + shared-file protocol; a crashing helper is an error on the effect, never a crash ([external-helpers.md](external-helpers.md), [ADR 0001](adr/0001-external-effect-helpers.md)). Native only | | L4 | `engine` | `Session`: project, branching undo history, editor state, the command registry and menus, the ScriptUI window model (`scriptui`) | | L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) and ScriptUI (dialogs, palettes, dockable panels; script-host threads so dialogs block `show()`), whose edits run engine commands; AE match names | diff --git a/docs/plugins.md b/docs/plugins.md index 3fd8cf944..290ed033c 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -11,7 +11,7 @@ Guide): | After Effects | EffectCraft | | |---|---|---| -| Effect plug-ins (C/C++ SDK, `.aex` / `.plugin`) | [Effect plug-ins](#effect-plug-ins): WebAssembly modules (plug-in API v1) or Rust; the `Plug-ins` folder next to the settings loads at start-up | Different ABI: After Effects SDK plug-ins can't run (G8) | +| Effect plug-ins (C/C++ SDK, `.aex` / `.plugin`) | [Effect plug-ins](#effect-plug-ins): WebAssembly modules (WebAssembly ABI 1 and 2) or Rust; the `Plug-ins` folder next to the settings loads at start-up | Different ABI: After Effects SDK plug-ins can't run (G8) | | Scripts folder: its scripts are listed in File ▸ Scripts | `Scripts/` in the settings folder (File ▸ Scripts ▸ Install Script File… copies a script there), listed in File ▸ Scripts with the bundled samples; a subfolder (one level) is a submenu named after it | Same | | File ▸ Scripts ▸ Run Script File… (`.jsx`, `.jsxbin`) | `.jsx` / `.js` | Gap: `.jsxbin` (an undocumented binary encoding) | | File ▸ Scripts ▸ Recent Script Files: the 10 most recently run scripts; Cmd+Option+Shift+D (Ctrl+Alt+Shift+D on Windows) reruns one | File ▸ Scripts ▸ Recent Script Files lists the 10 scripts run last (script files and File ▸ Scripts entries, newest first, kept in the settings); Cmd+Alt+Shift+D runs the most recent one again (`file.runScript {"recent": 0}`, a key that can be changed in Edit ▸ Keyboard Shortcuts) | Same | @@ -152,7 +152,7 @@ example of a keyframe tool. ## Effect plug-ins -EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 2**; WebAssembly ABI 1). +EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 2**; WebAssembly ABI 1 and 2). A plug-in effect behaves exactly like a built-in one: it is listed in Effects & Presets (and the Effect menu) under its category, applied by id (`effect.apply {"effect": "org.example.posterize"}`) or display name, its parameters are ordinary properties (keyframes, expressions, Effect Controls, @@ -178,7 +178,8 @@ author, source, params}]}`. * **Parameters** arrive flattened to `f64`s in the order they are declared: slider, angle, checkbox (0 / 1) and popup (0-based index) take one value, point two (already in **buffer pixels**), colour four (RGBA 0–1). -* **Time** is layer time in seconds. +* **Time** is layer time in seconds. A WebAssembly module written for ABI 2 also gets the + layer's neighbouring frames at other times (see [WebAssembly ABI](#webassembly-abi)). * Rendering must be **deterministic** and may run on several threads at once (a WebAssembly plug-in gets one instance per thread). * A failed render (an error code, a trap, running out of fuel) leaves the frame unchanged. @@ -216,22 +217,36 @@ Every plug-in describes itself with a JSON manifest: parameters; a slider needs `min ≤ default ≤ max` (and `sliderMin ≤ sliderMax`); the manifest is at most 1 MiB. -### WebAssembly ABI (v1) +### WebAssembly ABI -A module with **no imports** that exports: +A module with **no imports** exports: | export | signature | | |---|---|---| | `memory` | memory | linear memory shared with the host | -| `ec_api_version` | `() -> i32` | must return `1` | +| `ec_api_version` | `() -> i32` | `1` for ABI 1, `2` for ABI 2 (neighbouring frames) | | `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | the UTF-8 JSON manifest in memory | | `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | -| `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; return 0 on success | +| `ec_render` (ABI 1) | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; return 0 on success | +| `ec_render2` (ABI 2) | `ec_render`'s arguments, then `prev: i32, prev_time: f64, next: i32, next_time: f64, flags: i32` | the same, with the neighbouring frames; return 0 on success | For each frame the host calls `ec_alloc` once for a buffer holding the parameters (`nparams` -little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s), fills it, -calls `ec_render`, and reads the pixels back. Each call gets a fuel budget proportional to the -frame size; a module that loops forever stops with an error instead of hanging the app. +little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s, +premultiplied RGBA, row-major), fills it, calls the module's render export, and reads the pixels +back. An ABI 2 module gets two more pixel buffers after the current frame: the previous and next +frames, resampled onto the current buffer's grid. At a frame boundary they are the frames before +and after it (`time ± 1/fps`); between frames they are the two frames that bracket `time`. + +The neighbour buffers are always allocated (zeroed when the layer has no pixels there, so a +module can read both without a null check). `flags` bit 0 is set when `prev` holds a real frame, +bit 1 when `next` does; `prev_time` / `next_time` are those frames' layer times in seconds (0 +when absent). The ABI version is the module's `ec_api_version` and is independent of the +manifest's `api` (the native plug-in API): a module can use ABI 2 while its manifest says +`"api": 1`. Because an ABI 2 module reads other times, its layer is keyed on the layer time in +the render cache; ABI 1 modules keep the time-independent key. + +Each call gets a fuel budget proportional to the frame size; a module that loops forever stops +with an error instead of hanging the app. #### Example @@ -324,8 +339,10 @@ register_plugin(Arc::new(Swap(manifest)))?; // before the UI builds its menus ## Plug-in API 2 (native plug-ins) API 2 is for Rust plug-ins and for hosts that bring effects from elsewhere (such as -[external effect helpers](external-helpers.md)); the WebAssembly ABI stays at 1. Manifests with `"api": 1` or `2` are accepted, -`effect.plugins.list` reports `api: 2`, and up to 1024 plug-in effects can be registered. +[external effect helpers](external-helpers.md)); the WebAssembly ABI is versioned separately +(ABI 2 adds the neighbouring frames, see [WebAssembly ABI](#webassembly-abi)). Manifests with +`"api": 1` or `2` are accepted, `effect.plugins.list` reports `api: 2`, and up to 1024 plug-in +effects can be registered. * **New parameter types**: `point3` (`default` `[x, y, z]`; x, y arrive in buffer pixels, z as is), `layer` (another layer of the composition; flattens to its id, -1 for none, read with @@ -354,9 +371,11 @@ API 2 is for Rust plug-ins and for hosts that bring effects from elsewhere (such ### Compatibility -The API version is checked when a plug-in loads: a manifest or module for another version is -refused with a message. Within a version the contract above does not change; additions (new -parameter types, new optional exports) bump the version. +The API version is checked when a plug-in loads: this build implements plug-in API 1 and 2 and +WebAssembly ABI 1 and 2; a manifest or module for another version is refused with a message. +ABI 1 modules load and render unchanged; ABI 2 modules add `ec_render2` and the neighbouring +frames. Within a version the contract above does not change; additions (new parameter types, +new optional exports) bump the version. ## Plug-in Library diff --git a/examples/plugins/posterize-bands/Cargo.toml b/examples/plugins/posterize-bands/Cargo.toml index 825d704aa..65bbbaaaf 100644 --- a/examples/plugins/posterize-bands/Cargo.toml +++ b/examples/plugins/posterize-bands/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "posterize-bands" -description = "Example EffectCraft effect plug-in (WebAssembly, plug-in API v1): posterizes luminance into tinted bands" +description = "Example EffectCraft effect plug-in (WebAssembly ABI 1): posterizes luminance into tinted bands" version = "1.0.0" edition = "2024" license = "MIT OR Apache-2.0" diff --git a/examples/plugins/posterize-bands/src/lib.rs b/examples/plugins/posterize-bands/src/lib.rs index 9af709433..77026a16f 100644 --- a/examples/plugins/posterize-bands/src/lib.rs +++ b/examples/plugins/posterize-bands/src/lib.rs @@ -1,4 +1,4 @@ -//! Posterize Bands: an example EffectCraft effect plug-in (plug-in API v1, see +//! Posterize Bands: an example EffectCraft effect plug-in (WebAssembly ABI 1, see //! `docs/plugins.md`). It quantizes each pixel's luminance into `levels` bands and mixes a tint //! colour in by `mix` percent, optionally only above a `threshold`. //!