mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-29 12:37:08 -06:00
Cryptomatte pick: snap to the effect's input buffer grid, not the preview scale
At preview scales from 0.75 to 1 footage keeps native pixels, a proxy sets its own buffer scale at any render scale, a Preserve Resolution precomp is drawn at full size, and earlier effects can pad or offset the buffer, so snapping to the preview grid could pick a different ID than the rendered matte. Use the scale and offset of the buffer the Cryptomatte effect receives (layer_input, rendered once and then cached). The pixel centre goes to an aux point through effectcraft_effects::aux_point, which index_map_at now calls too, so the two round identically (a division vs the reciprocal picked the other ID on a 104 px proxy of 1920 px footage). An unknown effect uid errors "no such effect" and a layer without pixels "this layer has no pixels", as effect.pickColor does.
This commit is contained in:
Matej Kozlovský
@@ -37,16 +37,24 @@ fn index_map(b: &Buf, a: &AuxChannels) -> Vec<Option<usize>> {
|
||||
/// [`index_map`] sampling each pixel at sub-pixel position `at` (0..1).
|
||||
fn index_map_at(b: &Buf, a: &AuxChannels, at: [f64; 2]) -> Vec<Option<usize>> {
|
||||
let (w, h) = (b.img.width as usize, b.img.height as usize);
|
||||
let inv = 1.0 / b.scale.max(1e-9);
|
||||
(0..w * h)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let (x, y) = ((i % w) as f64 + at[0], (i / w) as f64 + at[1]);
|
||||
a.index_at((x - b.offset[0]) * inv, (y - b.offset[1]) * inv)
|
||||
let [x, y] = aux_point(b.scale, b.offset, [(i % w) as f64 + at[0], (i / w) as f64 + at[1]]);
|
||||
a.index_at(x, y)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The layer point where 3D Channel effects look up the aux for buffer position `p` (pixel
|
||||
/// `i` + sub-pixel offset) of a buffer with `scale` and `offset`. The Cryptomatte pick calls
|
||||
/// it too, so the two can't pick different aux pixels.
|
||||
#[inline]
|
||||
pub fn aux_point(scale: f64, offset: [f64; 2], p: [f64; 2]) -> [f64; 2] {
|
||||
let inv = 1.0 / scale.max(1e-9);
|
||||
[(p[0] - offset[0]) * inv, (p[1] - offset[1]) * inv]
|
||||
}
|
||||
|
||||
/// A channel sampled into the buffer grid (`fill` outside).
|
||||
fn plane_of(b: &Buf, idx: &[Option<usize>], ch: &[f32], fill: f32) -> Plane {
|
||||
Plane { w: b.img.width as usize, h: b.img.height as usize, data: idx.par_iter().map(|i| i.map_or(fill, |i| ch[i])).collect() }
|
||||
|
||||
@@ -111,7 +111,7 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time,
|
||||
// effectcraft-gpu fx_tone.
|
||||
pub use ocio::color_stabilizer_maps;
|
||||
// GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes).
|
||||
pub use channel3d::{CRYPTO_LAYERS, crypto_aux_image, id_color, selection_hashes};
|
||||
pub use channel3d::{CRYPTO_LAYERS, aux_point, crypto_aux_image, id_color, selection_hashes};
|
||||
pub use ocio::{ColorOp, Straight, Tf, color_program};
|
||||
pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, drizzle_drops, mercury_blobs, sprite_plan};
|
||||
pub use sim2::{
|
||||
|
||||
@@ -212,7 +212,8 @@ impl CryptoHit {
|
||||
|
||||
/// The ID under `{x, y}` (effect space, which is layer space on footage and precomp layers) for
|
||||
/// the Cryptomatte effect `{layer?, effect?}`, from the aux channels its render reads. The point
|
||||
/// snaps to the centre of the pixel a render at `scale` (the viewer's resolution) samples.
|
||||
/// snaps to the centre of the pixel the effect's input buffer samples in a render at `scale`
|
||||
/// (the viewer's resolution).
|
||||
fn crypto_hit(s: &Session, p: &Value, cmd: &str) -> Result<Option<CryptoHit>> {
|
||||
let (cid, lid, fx_uid) = find_fx(s, p, cmd)?;
|
||||
let (Some(x), Some(y)) = (f_p(p, "x"), f_p(p, "y")) else { return Err(bad(cmd, "missing `x` / `y` (layer pixels)")) };
|
||||
@@ -222,23 +223,32 @@ fn crypto_hit(s: &Session, p: &Value, cmd: &str) -> Result<Option<CryptoHit>> {
|
||||
}
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.uid == fx_uid)).ok_or_else(|| bad(cmd, "no such effect"))?;
|
||||
let fx = layer.effects().ok_or_else(|| bad(cmd, "no effects"))?;
|
||||
let pos = fx.groups().position(|g| g.uid == fx_uid).ok_or_else(|| bad(cmd, "no such effect"))?;
|
||||
let g = fx.groups().nth(pos).ok_or_else(|| bad(cmd, "no such effect"))?;
|
||||
let prop = g.prop("selection").filter(|_| g.match_id == "ec.3d.cryptomatte").ok_or_else(|| bad(cmd, "not a Cryptomatte effect"))?;
|
||||
let t = super::time_p(s, p, Some(comp));
|
||||
let ctx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None };
|
||||
let params = effectcraft_effects::flatten_params(g, &mut |pr| ctx.value(layer, pr));
|
||||
let aux = s.renderer(effectcraft_render::RenderOpts { scale, ..Default::default() }).layer_aux(&ctx, layer);
|
||||
let r = s.renderer(effectcraft_render::RenderOpts { scale, ..Default::default() });
|
||||
let aux = r.layer_aux(&ctx, layer);
|
||||
let want = effectcraft_effects::CRYPTO_LAYERS.get(params.e("layer") as usize).copied().unwrap_or("CryptoAsset");
|
||||
let Some((aux, layer_name)) = aux.and_then(|a| a.resolve_crypto_layer(want).map(|l| (a, l))) else {
|
||||
return Err(bad(cmd, "this layer has no Cryptomatte channels (a multi-layer OpenEXR with Cryptomatte, or a 3D precomp)"));
|
||||
};
|
||||
let snap = |v: f64| ((v * scale).floor() + 0.5) / scale;
|
||||
let Some(hash) = aux.index_at(snap(x), snap(y)).and_then(|i| aux.crypto_id_at(&layer_name, i)) else { return Ok(None) };
|
||||
// The grid of the effect's input buffer (footage keeps native pixels from 3/4 scale up,
|
||||
// proxies have their own size, earlier effects can pad it), rendered and cached.
|
||||
let b = r.layer_input(&ctx, layer, pos).ok_or_else(|| bad(cmd, "this layer has no pixels"))?;
|
||||
let (bs, o) = (b.scale, b.offset);
|
||||
// The centre of its pixel under the point, at the aux point the effect reads there.
|
||||
let [ax, ay] = effectcraft_effects::aux_point(bs, o, [(x * bs + o[0]).floor() + 0.5, (y * bs + o[1]).floor() + 0.5]);
|
||||
let Some(hash) = aux.index_at(ax, ay).and_then(|i| aux.crypto_id_at(&layer_name, i)) else { return Ok(None) };
|
||||
let (selection, driven) = (params.s("selection").to_string(), prop.expr.as_ref().is_some_and(|e| e.enabled));
|
||||
Ok(Some(CryptoHit { at: (cid, lid, t), aux, layer: layer_name, hash, selection, prop: prop.uid, driven }))
|
||||
}
|
||||
|
||||
/// `effect.cryptomatteAt`: the ID under a point, for the viewer's hover hint.
|
||||
/// `effect.cryptomatteAt`: the ID under a point, for the viewer's hover hint. The first call per
|
||||
/// frame renders the effect's input to learn its pixel grid (cached afterwards).
|
||||
fn crypto_at(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
Ok(crypto_hit(s, p, "effect.cryptomatteAt")?.map_or(Value::Null, |h| h.reply(&h.selection)))
|
||||
}
|
||||
|
||||
@@ -46,8 +46,8 @@ fn aux_at(t: Tick) -> AuxChannels {
|
||||
}
|
||||
|
||||
impl FootageSource for Crypto {
|
||||
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
Some(Arc::new(Image::new(8, 4)))
|
||||
fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
Some(Arc::new(Image::new(f.width, f.height)))
|
||||
}
|
||||
fn aux(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<AuxChannels>> {
|
||||
Some(Arc::new(aux_at(t)))
|
||||
@@ -58,11 +58,13 @@ struct Probe;
|
||||
impl crate::Importer for Probe {
|
||||
fn probe(&self, path: &str) -> Result<Footage, String> {
|
||||
let duration = Tick::from_seconds_f64(2.0);
|
||||
// A half-size proxy of the EXR.
|
||||
let (width, height) = if path.contains("proxy") { (4, 2) } else { (8, 4) };
|
||||
Ok(Footage {
|
||||
path: path.into(),
|
||||
kind: FootageKind::Video,
|
||||
width: 8,
|
||||
height: 4,
|
||||
width,
|
||||
height,
|
||||
has_video: true,
|
||||
frame_rate: FrameRate::from_f64(24.0),
|
||||
duration,
|
||||
@@ -179,3 +181,156 @@ fn picks_on_a_precomp_of_a_3d_comp() {
|
||||
let r = s.execute_checked("effect.pickCryptomatte", json!({"effect": 1, "x": 20, "y": 10})).unwrap();
|
||||
assert_eq!((r["name"].as_str(), r["selection"].as_str()), (Some("Red, Ball"), Some("\"Red, Ball\"")), "{r}");
|
||||
}
|
||||
|
||||
/// The matte the rendered Cryptomatte effect gives the picked ID at the buffer pixel under
|
||||
/// `{x, y}`: the pick has to name the ID the effect's own input buffer samples there.
|
||||
fn rendered_matte_of_pick(s: &mut Session, x: f64, y: f64, scale: f64) -> (String, f32) {
|
||||
let r = s.execute_checked("effect.pickCryptomatte", json!({"effect": "Cryptomatte", "x": x, "y": y, "scale": scale, "mode": "replace"})).unwrap();
|
||||
let fx = s.active_comp().unwrap().layers[0].effects().unwrap().groups().position(|g| g.match_id == "ec.3d.cryptomatte").unwrap() + 1;
|
||||
s.execute_checked("prop.set", json!({"path": format!("effects/#{fx}/matteOnly"), "value": true})).unwrap();
|
||||
let (cid, comp) = (s.active_comp_id().unwrap(), s.active_comp().unwrap());
|
||||
let layer = &comp.layers[0];
|
||||
let ctx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: Tick::ZERO, expr: None, footage: None };
|
||||
let buf = s.renderer(effectcraft_render::RenderOpts { scale, ..Default::default() }).content_buf(&ctx, layer).unwrap();
|
||||
let (px, py) = buf.to_px([x, y]);
|
||||
let m = buf.img.get(px.floor() as i64, py.floor() as i64)[0];
|
||||
(r["name"].as_str().unwrap_or("?").to_string(), m)
|
||||
}
|
||||
|
||||
/// Between 3/4 and full resolution footage keeps its native pixels: at 0.8 the effect samples
|
||||
/// layer pixel 2 (the comma name, 0.7) at x = 2.2, not the 0.8-grid pixel snapped back to 1.
|
||||
#[test]
|
||||
fn pick_at_scale_0_8_matches_the_native_buffer_the_effect_renders() {
|
||||
let mut s = session();
|
||||
let (name, m) = rendered_matte_of_pick(&mut s, 2.2, 1.5, 0.8);
|
||||
assert!(m >= 0.5, "picked {name:?}, but the render gives it matte {m} there");
|
||||
assert_eq!(name, COMMA);
|
||||
}
|
||||
|
||||
/// A half-size proxy renders a buffer at 0.5 layer pixels per pixel even at full resolution:
|
||||
/// x = 6.5 lies in buffer pixel 3, which samples layer x = 7 (the `left` / `right` tie).
|
||||
#[test]
|
||||
fn pick_on_a_proxy_matches_the_proxy_sized_buffer() {
|
||||
let mut s = session();
|
||||
let item = *s.project.items.iter().find(|(_, i)| matches!(i.kind, effectcraft_project::ItemKind::Footage(_))).unwrap().0;
|
||||
s.execute("file.setProxy", json!({"item": item.0, "path": "/renders/crypto_proxy.exr"})).unwrap();
|
||||
let (name, m) = rendered_matte_of_pick(&mut s, 6.5, 1.5, 1.0);
|
||||
assert!(m >= 0.5, "picked {name:?}, but the render gives it matte {m} there");
|
||||
assert_eq!(name, "left");
|
||||
}
|
||||
|
||||
/// 1920-px footage with `left` up to x = 120 and `right` after it, and a 104-px proxy.
|
||||
struct Edge;
|
||||
impl FootageSource for Edge {
|
||||
fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
Some(Arc::new(Image::new(f.width, f.height)))
|
||||
}
|
||||
fn aux(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<AuxChannels>> {
|
||||
let (w, h) = (1920usize, 40usize);
|
||||
let mut aux = AuxChannels::new(w as u32, h as u32, 1.0);
|
||||
let ids = (0..w * h).map(|i| crypto_float(crypto_hash(if i % w < 120 { "left" } else { "right" }))).collect();
|
||||
aux.channels = vec![("CryptoObject00.R".into(), ids), ("CryptoObject00.G".into(), vec![1.0; w * h])];
|
||||
aux.manifests = vec![("CryptoObject".into(), ["left", "right"].iter().map(|n| (n.to_string(), crypto_hash(n))).collect())];
|
||||
Some(Arc::new(aux))
|
||||
}
|
||||
}
|
||||
|
||||
/// Proxy buffer pixel 6's centre maps to layer x = 6.5 / (104 / 1920): 120.0 by division, but
|
||||
/// 119.99999999999999 by the reciprocal the effect multiplies with. The pick must agree.
|
||||
#[test]
|
||||
fn pick_maps_pixel_centres_bit_exactly_as_the_effect() {
|
||||
let mut s = Session { footage: Arc::new(Edge), ..Default::default() };
|
||||
let at = |path: &str, width| Footage {
|
||||
path: path.into(),
|
||||
kind: FootageKind::Video,
|
||||
width,
|
||||
height: 40,
|
||||
has_video: true,
|
||||
frame_rate: FrameRate::from_f64(24.0),
|
||||
duration: Tick::from_seconds_f64(2.0),
|
||||
..Default::default()
|
||||
};
|
||||
s.importer = Some(Arc::new(EdgeProbe(at("/renders/edge.exr", 1920), at("/renders/edge_proxy.exr", 104))));
|
||||
let item = s.execute("file.import", json!({"paths": ["/renders/edge.exr"]})).unwrap()["items"][0].clone();
|
||||
s.execute("comp.new", json!({"name": "C", "width": 1920, "height": 40, "frameRate": 24, "duration": 2})).unwrap();
|
||||
s.execute("layer.addItem", json!({"item": item})).unwrap();
|
||||
s.execute_checked("effect.apply", json!({"effect": "Cryptomatte"})).unwrap();
|
||||
s.execute("file.setProxy", json!({"item": item, "path": "/renders/edge_proxy.exr"})).unwrap();
|
||||
let (name, m) = rendered_matte_of_pick(&mut s, 118.0, 5.0, 1.0);
|
||||
assert_eq!((name.as_str(), m), ("left", 1.0));
|
||||
}
|
||||
|
||||
struct EdgeProbe(Footage, Footage);
|
||||
impl crate::Importer for EdgeProbe {
|
||||
fn probe(&self, path: &str) -> Result<Footage, String> {
|
||||
Ok(if path.contains("proxy") { self.1.clone() } else { self.0.clone() })
|
||||
}
|
||||
}
|
||||
|
||||
/// Footage whose decoded frames are `num / den` of the probed size (a file re-rendered at another
|
||||
/// resolution, a relinked file): the render stretches them over the layer.
|
||||
struct OffNominal(u32, u32);
|
||||
impl FootageSource for OffNominal {
|
||||
fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
Some(Arc::new(Image::new(f.width * self.0 / self.1, f.height * self.0 / self.1)))
|
||||
}
|
||||
fn aux(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<AuxChannels>> {
|
||||
Some(Arc::new(aux_at(t)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Points along rows `ys` where the picked ID isn't lit in the rendered matte under the point.
|
||||
fn mismatches(s: &mut Session, scales: &[f64], xs: impl Iterator<Item = f64> + Clone, ys: &[f64]) -> Vec<String> {
|
||||
let mut bad = vec![];
|
||||
for &scale in scales {
|
||||
for (x, &y) in ys.iter().flat_map(|y| xs.clone().map(move |x| (x, y))) {
|
||||
if s.execute("effect.cryptomatteAt", json!({"effect": "Cryptomatte", "x": x, "y": y, "scale": scale})).unwrap().is_null() {
|
||||
continue;
|
||||
}
|
||||
let (name, m) = rendered_matte_of_pick(s, x, y, scale);
|
||||
if m < 0.5 {
|
||||
bad.push(format!("scale {scale} ({x:.2}, {y}): picked {name:?}, matte {m}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
bad
|
||||
}
|
||||
|
||||
const PREVIEW_SCALES: [f64; 7] = [1.0, 0.8, 0.75, 0.74, 0.5, 0.33, 0.25];
|
||||
|
||||
/// The pick follows the decoded frame's real size, not the footage's nominal one.
|
||||
#[test]
|
||||
fn pick_matches_the_render_when_frames_decode_off_nominal_size() {
|
||||
for (num, den) in [(2, 1), (1, 2), (3, 4)] {
|
||||
let mut s = Session { footage: Arc::new(OffNominal(num, den)), importer: Some(Arc::new(Probe)), ..Default::default() };
|
||||
let item = s.execute("file.import", json!({"paths": ["/renders/crypto.exr"]})).unwrap()["items"][0].clone();
|
||||
s.execute("comp.new", json!({"name": "C", "width": 8, "height": 4, "frameRate": 24, "duration": 2})).unwrap();
|
||||
s.execute("layer.addItem", json!({"item": item})).unwrap();
|
||||
s.execute_checked("effect.apply", json!({"effect": "Cryptomatte"})).unwrap();
|
||||
let bad = mismatches(&mut s, &PREVIEW_SCALES, (0..80).map(|i| i as f64 * 0.1 + 0.05), &[0.3, 1.7, 2.6]);
|
||||
assert!(bad.is_empty(), "decode {num}/{den}: {} mismatches:\n{}", bad.len(), bad.join("\n"));
|
||||
}
|
||||
}
|
||||
|
||||
/// A precomp of a 3D comp, with and without Preserve Resolution (drawn at full size below full
|
||||
/// resolution), behind a disabled effect: the pick names the ID the rendered matte shows.
|
||||
#[test]
|
||||
fn pick_matches_the_render_on_precomps_with_and_without_preserve_resolution() {
|
||||
for preserve in [false, true] {
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "Outer", "width": 40, "height": 20, "frameRate": 24, "duration": 2})).unwrap();
|
||||
let a = s.execute("layer.newSolid", json!({"name": "A", "width": 40, "height": 20})).unwrap()["layer"].clone();
|
||||
let b = s.execute("layer.newSolid", json!({"name": "B", "width": 7, "height": 20})).unwrap()["layer"].clone();
|
||||
s.execute_checked("prop.set", json!({"layer": b, "path": "transform/position", "value": [3.5, 10, -10]})).unwrap();
|
||||
s.execute("layer.setSwitch", json!({"layers": [a, b], "switch": "threeD", "value": true})).unwrap();
|
||||
s.execute("layer.precompose", json!({"layers": [a, b], "name": "Inner"})).unwrap();
|
||||
let inner = *s.project.items.iter().find(|(_, i)| i.name == "Inner").unwrap().0;
|
||||
s.execute_checked("comp.settings", json!({"comp": inner.0, "preserveResolution": preserve})).unwrap();
|
||||
s.execute("layer.select", json!({"layers": [s.active_comp().unwrap().layers[0].id.0]})).ok();
|
||||
s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur"})).unwrap();
|
||||
s.execute_checked("effect.toggle", json!({"effect": 1, "value": false})).unwrap();
|
||||
s.execute_checked("effect.apply", json!({"effect": "Cryptomatte"})).unwrap();
|
||||
let bad = mismatches(&mut s, &PREVIEW_SCALES, (0..93).map(|i| i as f64 * 0.15 + 0.05), &[10.0]);
|
||||
assert!(bad.is_empty(), "preserve {preserve}: {} mismatches:\n{}", bad.len(), bad.join("\n"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user