Track Matte: set the matte of several layers at once (#1260)

layer.setTrackMatte took one layer. It now also takes `layers`, set in
one undo step; `layer` and the first selected layer still work as
before. `kind: "preserve"` keeps each layer's own matte mode (Alpha for
a layer without one), so giving several layers a new matte doesn't turn
a mix of Alpha and Luma mattes into one mode; without `kind` the mode
is Alpha, as before. A layer can't be its own matte and an empty list
is refused, before anything changes.

Test: several_layers_take_one_matte_and_keep_their_modes.

crates/engine/src/commands/layer.rs

@@ -551,32 +551,47 @@ fn blend(s: &mut Session, p: &Value) -> Result<Value> {
     Ok(Value::Null)
 }
 
+/// `layer.setTrackMatte`: `layers` (or `layer`, else the first selected layer) use `matte` as
+/// their track matte (none when null), in one undo step. `kind` sets the mode; `preserve` keeps
+/// each layer's own (Alpha for a layer without a matte), so changing the matte of several layers
+/// at once keeps their modes.
 fn track_matte(s: &mut Session, p: &Value) -> Result<Value> {
-    let (cid, lid) = layer_p(s, p, "layer.setTrackMatte")?;
+    let (cid, ids) = match p.get("layers") {
+        Some(_) => layers_p(s, p)?,
+        None => layer_p(s, p, "layer.setTrackMatte").map(|(c, l)| (c, vec![l]))?,
+    };
     let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
     let matte = match p.get("matte") {
         None | Some(Value::Null) => None,
         Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad("layer.setTrackMatte", "no such matte layer"))?),
     };
-    let kind = str_p(p, "kind")
-        .map(|k| match k.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() {
+    if ids.is_empty() {
+        return Err(bad("layer.setTrackMatte", "no layers"));
+    }
+    if matte.is_some_and(|m| ids.contains(&m)) {
+        return Err(bad("layer.setTrackMatte", "a layer can't be its own matte"));
+    }
+    // None: each layer keeps its own mode.
+    let kind = match str_p(p, "kind").map(|k| k.to_ascii_lowercase().replace([' ', '_', '-'], "")) {
+        Some(k) if k == "preserve" => None,
+        Some(k) => Some(match k.as_str() {
             "alphainverted" | "alphainv" => MatteKind::AlphaInverted,
             "luma" => MatteKind::Luma,
             "lumainverted" | "lumainv" => MatteKind::LumaInverted,
             _ => MatteKind::Alpha,
-        })
-        .unwrap_or(MatteKind::Alpha);
+        }),
+        None => Some(MatteKind::Alpha),
+    };
     s.edit("Track Matte", None, |proj, _| {
         let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?;
-        if let Some(m) = matte {
-            if m == lid {
-                return Err(bad("layer.setTrackMatte", "a layer can't be its own matte"));
-            }
-            if let Some(ml) = comp.layer_mut(m) {
-                ml.switches.video = false;
-            }
+        if let Some(ml) = matte.and_then(|m| comp.layer_mut(m)) {
+            ml.switches.video = false;
+        }
+        for lid in &ids {
+            let l = comp.layer_mut(*lid).ok_or(EngineError::NoComp)?;
+            let kind = kind.unwrap_or_else(|| l.track_matte.map(|m| m.kind).unwrap_or_default());
+            l.track_matte = matte.map(|layer| TrackMatte { layer, kind });
         }
-        comp.layer_mut(lid).ok_or(EngineError::NoComp)?.track_matte = matte.map(|layer| TrackMatte { layer, kind });
         Ok(())
     })?;
     Ok(Value::Null)
@@ -1656,7 +1671,7 @@ pub fn specs() -> Vec<CommandSpec> {
             "Track Matte",
             [],
             None,
-            "{layer?, matte: layer|null, kind?: alpha|alphaInverted|luma|lumaInverted}",
+            "{layers? | layer?, matte: layer|null, kind?: alpha|alphaInverted|luma|lumaInverted|preserve}",
             has_layers,
             track_matte
         ),
@@ -1862,3 +1877,49 @@ mod parent_chain_tests {
         assert!(check_parent_chain(&comp, &[LayerId(c)], Some(LayerId(10_000)), 4096, 1_048_576).is_ok());
     }
 }
+
+#[cfg(test)]
+mod track_matte_batch_tests {
+    use super::*;
+    use std::sync::Arc;
+
+    fn matte_of(s: &Session, id: u64) -> Option<(u64, MatteKind)> {
+        s.active_comp().unwrap().layer(LayerId(id)).unwrap().track_matte.map(|m| (m.layer.0, m.kind))
+    }
+
+    /// #1260: several layers take one matte layer at once, in one undo step; `kind: "preserve"`
+    /// keeps each one's mode; refused calls change nothing; one `layer` still works.
+    #[test]
+    fn several_layers_take_one_matte_and_keep_their_modes() {
+        let mut s = Session::default();
+        s.execute("comp.new", json!({"name":"Mattes","width":16,"height":16,"duration":1})).unwrap();
+        let [a, b, c, x, y] = ["A", "B", "C", "X", "Y"].map(|n| s.execute("layer.newSolid", json!({"name": n})).unwrap()["layer"].as_u64().unwrap());
+        s.execute("layer.setTrackMatte", json!({"layer": a, "matte": x, "kind": "luma"})).unwrap();
+        s.execute("layer.setTrackMatte", json!({"layer": b, "matte": x})).unwrap();
+        assert_eq!((matte_of(&s, a), matte_of(&s, b)), (Some((x, MatteKind::Luma)), Some((x, MatteKind::Alpha))), "one layer, as before");
+        let (before, undo) = (s.project.clone(), s.history.undo.len());
+        s.execute("layer.setTrackMatte", json!({"layers": [a, b, c], "matte": y, "kind": "preserve"})).unwrap();
+        assert_eq!(
+            [matte_of(&s, a), matte_of(&s, b), matte_of(&s, c)],
+            [Some((y, MatteKind::Luma)), Some((y, MatteKind::Alpha)), Some((y, MatteKind::Alpha))],
+            "each keeps its mode (Alpha without one)"
+        );
+        assert!(!s.active_comp().unwrap().layer(LayerId(y)).unwrap().switches.video, "the matte is hidden");
+        assert_eq!(s.history.undo.len(), undo + 1, "one undo step");
+        s.execute("edit.undo", json!({})).unwrap();
+        assert!(s.project.same_content(&before));
+        // Refused: a layer as its own matte, an unknown layer, no layers. Nothing changes.
+        for p in [
+            json!({"layers": [a, y], "matte": y}),
+            json!({"layers": [a, 9999], "matte": y}),
+            json!({"layers": [], "matte": y}),
+            json!({"layers": [a], "matte": 9999}),
+        ] {
+            let (project, revision) = (s.project.clone(), s.revision);
+            assert!(s.execute("layer.setTrackMatte", p.clone()).is_err(), "{p}");
+            assert!(Arc::ptr_eq(&project, &s.project) && s.revision == revision, "{p} changed nothing");
+        }
+        s.execute("layer.setTrackMatte", json!({"layers": [a, b], "matte": null})).unwrap();
+        assert_eq!((matte_of(&s, a), matte_of(&s, b)), (None, None), "both lose it");
+    }
+}