#813: masks round their corners (rounded rectangle masks)

Problem: Premiere 26's redesigned mask tools give rectangle masks
rounded corners; FilmCraft masks only had sharp corners.

Fix: MaskPath::with_rounded_corners replaces each sharp corner between
straight edges by a circular fillet (two vertices joined by a cubic arc,
handles 4/3·tan(φ/4)·r, shrunk to fit half of each edge; hostile radii
and indices are clamped or ignored). masks.add takes `cornerRadius` for
a rounded rectangle, and the new masks.roundCorners command rounds an
existing mask's corners (or the listed vertices) in one undo step, on
every Mask Path keyframe so animated paths keep interpolating.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/masks.rs

@@ -38,7 +38,7 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
         spec(
             "masks.add",
             "Create Mask",
-            r#"{"clip":id?,"effect":index|id?="opacity","shape":"ellipse"|"polygon"|"bezier","path":[[x,y]…]|{vertices}?,"center":[x,y]?,"size":[w,h]?}"#,
+            r#"{"clip":id?,"effect":index|id?="opacity","shape":"ellipse"|"polygon"|"rectangle"|"bezier","path":[[x,y]…]|{vertices}?,"center":[x,y]?,"size":[w,h]?,"cornerRadius":px?}"#,
             add,
         ),
         spec("masks.remove", "Delete Mask", r#"{"clip":id?,"effect":index|id?,"mask":n?}"#, remove),
@@ -64,6 +64,7 @@ pub(crate) fn commands() -> Vec<CommandSpec> {
             track,
         ),
         spec("masks.toggleVertexSmooth", "Convert Mask Vertex", r#"{"clip":id?,"effect":index|id?,"mask":n?,"vertex":n}"#, toggle_smooth),
+        spec("masks.roundCorners", "Round Mask Corners", r#"{"clip":id?,"effect":index|id?,"mask":n?,"radius":px,"vertices":[n]?}"#, round_corners),
         CommandSpec {
             id: "masks.select",
             label: "Select Mask",
@@ -212,7 +213,12 @@ fn add(s: &mut Session, p: &Value) -> Result<Value> {
         None => match shape.as_str() {
             "ellipse" => MaskPath::ellipse(center, size * 0.5),
             "polygon" | "rectangle" | "rect" | "4-point" => {
-                MaskPath::rect(center.x - size.x / 2.0, center.y - size.y / 2.0, center.x + size.x / 2.0, center.y + size.y / 2.0)
+                let (x0, y0, x1, y1) = (center.x - size.x / 2.0, center.y - size.y / 2.0, center.x + size.x / 2.0, center.y + size.y / 2.0);
+                match f64_p(p, "cornerRadius") {
+                    Some(r) if !r.is_finite() || r < 0.0 => return Err(bad("masks.add", "`cornerRadius` is pixels ≥ 0")),
+                    Some(r) => MaskPath::rounded_rect(x0, y0, x1, y1, r),
+                    None => MaskPath::rect(x0, y0, x1, y1),
+                }
             }
             _ => return Err(bad("masks.add", "a bezier mask needs `path` (pen points)")),
         },
@@ -454,6 +460,40 @@ fn toggle_smooth(s: &mut Session, p: &Value) -> Result<Value> {
     Ok(Value::Null)
 }
 
+/// Round the mask's sharp corners (Premiere 26's rounded rectangle masks): every corner between
+/// two straight edges (or only the listed `vertices`) becomes a circular fillet of `radius`
+/// pixels, shrunk to fit half of each edge. The same corners are rounded in every Mask Path
+/// keyframe so an animated path keeps interpolating. Returns how many corners were rounded.
+fn round_corners(s: &mut Session, p: &Value) -> Result<Value> {
+    const ID: &str = "masks.roundCorners";
+    let radius = f64_p(p, "radius").filter(|r| r.is_finite() && *r > 0.0).ok_or_else(|| bad(ID, "need `radius` (pixels, > 0)"))?;
+    let only: Option<Vec<usize>> = match p.get("vertices") {
+        None | Some(Value::Null) => None,
+        Some(v) => {
+            let list = v.as_array().ok_or_else(|| bad(ID, "`vertices` is a list of vertex indices"))?;
+            let mut out = list
+                .iter()
+                .map(|i| i.as_u64().and_then(|i| usize::try_from(i).ok()).ok_or_else(|| bad(ID, "`vertices` is a list of vertex indices")))
+                .collect::<Result<Vec<_>>>()?;
+            out.sort_unstable();
+            out.dedup();
+            Some(out)
+        }
+    };
+    let rounded = edit_mask(s, p, "Round Mask Corners", move |m, t| {
+        let mut corners = m.path_at(t).roundable_corners();
+        if let Some(only) = &only {
+            corners.retain(|i| only.binary_search(i).is_ok());
+        }
+        if corners.is_empty() {
+            return Err(bad(ID, "no sharp corner between straight edges to round"));
+        }
+        drop_all(m, t, |path| *path = path.with_rounded_corners(radius, &corners));
+        Ok(corners.len())
+    })?;
+    Ok(json!({"corners": rounded}))
+}
+
 fn select(s: &mut Session, p: &Value) -> Result<Value> {
     if bool_p(p, "none") == Some(true) || p.is_null() || p.as_object().is_some_and(|o| o.is_empty()) {
         s.state.selected_mask = None;

crates/engine/src/masks_tests.rs

@@ -317,3 +317,66 @@ fn tracking_runs_in_the_background_and_can_be_cancelled() {
     assert!(n < 48, "stopped early ({n} keyframes)");
     assert!(s.execute("masks.track", json!({"direction": "backward"})).is_err(), "nothing before the first frame");
 }
+
+/// #813: a rectangle mask with rounded corners (created rounded, or rounded afterwards) cuts the
+/// clip's corners away while the edges stay where they were; one undo step, every keyframe.
+#[test]
+fn rounded_rectangle_masks_cut_their_corners() {
+    let (mut s, clip) = demo();
+    // a 960×540 opacity mask in the middle of the 1920×1080 frame, rounded by 200 px
+    s.execute("masks.add", json!({"clip": clip.0, "shape": "rectangle", "center": [960, 540], "size": [960, 540], "cornerRadius": 200})).unwrap();
+    s.execute("masks.set", json!({"feather": 0})).unwrap();
+    let list = s.execute("masks.list", json!({})).unwrap();
+    assert_eq!(list["masks"][0]["path"]["vertices"].as_array().unwrap().len(), 8);
+    let img = s.render_program(0.25).unwrap();
+    // just inside the rectangle's corner: cut away; middle of an edge and the centre: kept
+    let corner = px(&img, 0.26, 0.26);
+    assert!(corner[0] + corner[1] + corner[2] < 1e-4, "rounded corner cut: {corner:?}");
+    assert!(px(&img, 0.5, 0.26)[3] > 0.99, "top edge kept");
+    assert!(px(&img, 0.5, 0.5)[3] > 0.99, "inside kept");
+
+    // round an existing 4-point mask: same picture, one undo step back to sharp corners
+    s.undo();
+    s.undo();
+    s.execute("masks.add", json!({"clip": clip.0, "shape": "polygon", "center": [960, 540], "size": [960, 540]})).unwrap();
+    s.execute("masks.set", json!({"feather": 0})).unwrap();
+    let sharp = s.render_program(0.25).unwrap();
+    assert!(px(&sharp, 0.26, 0.26)[3] > 0.99, "a sharp corner keeps the clip");
+    let r = s.execute("masks.roundCorners", json!({"radius": 200})).unwrap();
+    assert_eq!(r["corners"], 4);
+    assert_eq!(s.render_program(0.25).unwrap().px, img.px, "rounding afterwards = creating rounded");
+    // nothing left to round; bad radii are refused without changing the mask
+    assert!(s.execute("masks.roundCorners", json!({"radius": 50})).is_err());
+    for radius in [json!(0), json!(-5), json!("big"), json!(null)] {
+        assert!(s.execute("masks.roundCorners", json!({"radius": radius})).is_err(), "{radius}");
+    }
+    s.undo();
+    assert_eq!(s.render_program(0.25).unwrap().px, sharp.px, "one undo step restores the sharp corners");
+}
+
+#[test]
+fn rounding_an_animated_mask_rounds_every_keyframe_and_only_the_chosen_corners() {
+    let (mut s, clip) = demo();
+    let it = s.active_sequence().unwrap().find_item(clip).unwrap().1.clone();
+    s.set_playhead(it.start);
+    s.execute("masks.add", json!({"effect": "opacity", "shape": "polygon", "center": [400, 400], "size": [200, 200]})).unwrap();
+    s.execute("effects.toggleAnimation", json!({"clip": clip.0, "effect": "opacity", "mask": 0, "param": "path"})).unwrap();
+    let rate = s.active_sequence().unwrap().settings.frame_rate;
+    s.set_playhead(it.start + rate.tick_of(10));
+    s.execute("masks.translate", json!({"delta": [300, 0]})).unwrap();
+    // only vertices 0 and 2 (top-left, bottom-right), out-of-range indices ignored
+    let r = s.execute("masks.roundCorners", json!({"radius": 30, "vertices": [0, 2, 99]})).unwrap();
+    assert_eq!(r["corners"], 2);
+    let m = &s.active_sequence().unwrap().find_item(clip).unwrap().1.effects.iter().find(|e| e.effect == "opacity").unwrap().masks[0];
+    assert_eq!(m.path.keyframes.len(), 2);
+    for k in &m.path.keyframes {
+        let filmcraft_project::ParamValue::Path(p) = &k.value else { panic!("path keyframe") };
+        assert_eq!(p.len(), 6, "both keyframes rounded the same corners");
+        assert_eq!(p.vertices.iter().filter(|v| v.is_corner()).count(), 2, "vertices 1 and 3 stay sharp");
+    }
+    // halfway the path still interpolates (equal vertex counts)
+    let l = s.execute("masks.list", json!({"time": (it.start + rate.tick_of(5)).0})).unwrap();
+    let v = l["masks"][0]["path"]["vertices"].as_array().unwrap();
+    assert_eq!(v.len(), 6);
+    assert!((v[0]["p"][0].as_f64().unwrap() - 450.0).abs() < 1e-6, "{}", v[0]);
+}

crates/project/src/mask.rs

@@ -1,9 +1,12 @@
-//! Effect masks (Premiere's per-effect masks: ellipse, 4-point polygon, free-draw Bézier).
+//! Effect masks (Premiere's per-effect masks: ellipse, 4-point polygon or rounded rectangle,
+//! free-draw Bézier).
 //!
 //! Every mask is a closed cubic Bézier path in **clip pixel space** (full-resolution source
 //! pixels, the space of effect point parameters). An ellipse is four smooth vertices with circular
-//! tangents and a polygon is vertices with zero-length tangents, so all three tools produce the same
-//! [`MaskPath`] and the same rasterizer, keyframe interpolation and tracker apply to each.
+//! tangents, a polygon is vertices with zero-length tangents and a rounded corner is a pair of
+//! vertices joined by a circular-arc cubic ([`MaskPath::with_rounded_corners`]), so all the tools
+//! produce the same [`MaskPath`] and the same rasterizer, keyframe interpolation and tracker apply
+//! to each.
 //!
 //! A mask has a keyframable *Mask Path* (vertex-wise interpolation when the vertex counts match,
 //! hold otherwise), *Feather* (falloff width in pixels, centred on the edge), *Opacity* (%),
@@ -80,6 +83,86 @@ impl MaskPath {
         Self::polygon(&[Vec2::new(x0, y0), Vec2::new(x1, y0), Vec2::new(x1, y1), Vec2::new(x0, y1)])
     }
 
+    /// An axis-aligned rectangle with corners rounded by `radius` (Premiere 26's rounded
+    /// rectangle mask). `radius` ≤ 0 (or not finite) gives the plain 4-point [`MaskPath::rect`].
+    pub fn rounded_rect(x0: f64, y0: f64, x1: f64, y1: f64, radius: f64) -> Self {
+        let r = Self::rect(x0, y0, x1, y1);
+        if radius.is_finite() && radius > 0.0 { r.with_rounded_corners(radius, &[0, 1, 2, 3]) } else { r }
+    }
+
+    /// Indices of the sharp corners [`MaskPath::with_rounded_corners`] can round: corner vertices
+    /// of a closed path whose two segments are straight lines.
+    pub fn roundable_corners(&self) -> Vec<usize> {
+        let n = self.vertices.len();
+        if !self.closed || n < 3 {
+            return Vec::new();
+        }
+        (0..n)
+            .filter(|&i| {
+                let (Some(prev), Some(v), Some(next)) = (self.vertices.get((i + n - 1) % n), self.vertices.get(i), self.vertices.get((i + 1) % n)) else {
+                    return false;
+                };
+                v.is_corner() && is_zero(&prev.t_out) && is_zero(&next.t_in)
+            })
+            .collect()
+    }
+
+    /// The path with each listed corner replaced by a circular fillet of `radius`: two vertices on
+    /// the adjacent edges joined by a cubic arc (handles `4/3·tan(φ/4)·r` for a turn of φ, so a
+    /// right angle gets the [`KAPPA`] quarter circle). The fillet is shrunk so it never takes more
+    /// than half of either edge. Every listed roundable corner ([`MaskPath::roundable_corners`])
+    /// becomes two vertices, coincident when its geometry is degenerate (a straight or reversing
+    /// corner, a zero-length edge), so applying the same list to each keyframe of an animated path
+    /// keeps the vertex counts equal. Other indices (out of range, curved, repeated) are left alone.
+    pub fn with_rounded_corners(&self, radius: f64, corners: &[usize]) -> Self {
+        let n = self.vertices.len();
+        let mut round = vec![false; n];
+        for &i in corners {
+            if let Some(r) = round.get_mut(i) {
+                *r = true;
+            }
+        }
+        let roundable = self.roundable_corners();
+        round.iter_mut().enumerate().for_each(|(i, r)| *r &= roundable.binary_search(&i).is_ok());
+        let radius = if radius.is_finite() { radius.clamp(0.0, 1e7) } else { 0.0 };
+        let mut vertices = Vec::with_capacity(n.saturating_mul(2));
+        for (i, v) in self.vertices.iter().enumerate() {
+            if !round.get(i).copied().unwrap_or(false) {
+                vertices.push(*v);
+                continue;
+            }
+            let (Some(prev), Some(next)) = (self.vertices.get((i + n - 1) % n), self.vertices.get((i + 1) % n)) else {
+                vertices.push(*v);
+                continue;
+            };
+            let (a, b) = (v.p - prev.p, next.p - v.p);
+            let (la, lb) = (a.length(), b.length());
+            let fillet = (la > 1e-9 && lb > 1e-9 && radius > 0.0).then(|| (a * (1.0 / la), b * (1.0 / lb))).and_then(|(u, w)| {
+                // turn angle φ between the incoming and outgoing directions
+                let phi = (u.x * w.x + u.y * w.y).clamp(-1.0, 1.0).acos();
+                if !(1e-6..std::f64::consts::PI - 1e-6).contains(&phi) {
+                    return None;
+                }
+                let tan_half = (phi / 2.0).tan();
+                let d = (radius * tan_half).min(la / 2.0).min(lb / 2.0);
+                let r = d / tan_half;
+                let h = 4.0 / 3.0 * (phi / 4.0).tan() * r;
+                Some((u, w, d, h))
+            });
+            match fillet {
+                Some((u, w, d, h)) => {
+                    vertices.push(MaskVertex { p: v.p - u * d, t_in: Vec2::ZERO, t_out: u * h });
+                    vertices.push(MaskVertex { p: v.p + w * d, t_in: w * -h, t_out: Vec2::ZERO });
+                }
+                None => {
+                    vertices.push(MaskVertex::corner(v.p));
+                    vertices.push(MaskVertex::corner(v.p));
+                }
+            }
+        }
+        Self { vertices, closed: self.closed }
+    }
+
     pub fn len(&self) -> usize {
         self.vertices.len()
     }
@@ -439,4 +522,59 @@ mod tests {
         let back: Mask = serde_json::from_str(&s).unwrap();
         assert_eq!(back, m);
     }
+
+    #[test]
+    fn rounded_rect_corners_are_quarter_circles() {
+        let p = MaskPath::rounded_rect(0.0, 0.0, 100.0, 60.0, 20.0);
+        assert_eq!(p.len(), 8);
+        // top-left corner: a quarter circle of radius 20 about (20, 20); the edges stay straight
+        let pts = p.flatten(0.01);
+        let mut on_arc = 0;
+        for q in &pts {
+            let (cx, cy) = (q.x.clamp(20.0, 80.0), q.y.clamp(20.0, 40.0));
+            let d = ((q.x - cx).powi(2) + (q.y - cy).powi(2)).sqrt();
+            if (q.x - cx).abs() > 1e-9 && (q.y - cy).abs() > 1e-9 {
+                on_arc += 1;
+                assert!((d - 20.0).abs() < 0.03, "{q:?} is {d} from its corner centre");
+            }
+            assert!(q.x >= -1e-9 && q.x <= 100.0 + 1e-9 && q.y >= -1e-9 && q.y <= 60.0 + 1e-9, "{q:?} outside the rectangle");
+        }
+        assert!(on_arc >= 8, "the corners are curved: {on_arc} arc points");
+        // the sharp corner itself is cut away
+        assert!(pts.iter().all(|q| q.x + q.y > 5.0), "the corner at (0, 0) is rounded off");
+        // radius 0 is the plain rectangle
+        assert_eq!(MaskPath::rounded_rect(0.0, 0.0, 10.0, 5.0, 0.0), MaskPath::rect(0.0, 0.0, 10.0, 5.0));
+    }
+
+    #[test]
+    fn rounding_clamps_to_half_an_edge_and_ignores_curves_and_hostile_input() {
+        // a radius larger than the shape gives a stadium: fillets meet at the edges' midpoints
+        let p = MaskPath::rounded_rect(0.0, 0.0, 100.0, 40.0, 1e9);
+        assert_eq!(p.len(), 8);
+        for v in &p.vertices {
+            assert!(v.p.x.is_finite() && v.p.y.is_finite());
+        }
+        assert!(p.vertices.iter().any(|v| (v.p - Vec2::new(0.0, 20.0)).length() < 1e-9), "{:?}", p.vertices);
+        // a triangle: the 60° turns get fillets tangent to both edges, within the triangle
+        let tri = MaskPath::polygon(&[Vec2::new(0.0, 0.0), Vec2::new(100.0, 0.0), Vec2::new(50.0, 86.6)]);
+        let r = tri.with_rounded_corners(10.0, &tri.roundable_corners());
+        assert_eq!(r.len(), 6);
+        assert!(r.vertices.iter().all(|v| v.p.y >= -1e-9));
+        // an ellipse has no sharp corners; NaN / negative radii and bad indices change nothing harmful
+        let e = MaskPath::ellipse(Vec2::new(50.0, 50.0), Vec2::new(30.0, 20.0));
+        assert!(e.roundable_corners().is_empty());
+        assert_eq!(e.with_rounded_corners(10.0, &[0, 1, 2, 3]), e);
+        let sq = MaskPath::rect(0.0, 0.0, 10.0, 10.0);
+        assert_eq!(sq.with_rounded_corners(5.0, &[usize::MAX, 99]), sq);
+        for bad in [f64::NAN, f64::INFINITY, -3.0, 0.0] {
+            let q = sq.with_rounded_corners(bad, &[0]);
+            assert_eq!(q.len(), 5, "a listed corner always splits in two");
+            assert!(q.vertices.iter().all(|v| v.is_corner() && v.p.x.is_finite()));
+        }
+        // a collinear vertex splits into two coincident corners, keeping vertex counts stable
+        let line = MaskPath::polygon(&[Vec2::new(0.0, 0.0), Vec2::new(5.0, 0.0), Vec2::new(10.0, 0.0), Vec2::new(5.0, 5.0)]);
+        let q = line.with_rounded_corners(2.0, &[1]);
+        assert_eq!(q.len(), 5);
+        assert_eq!(q.vertices[1].p, q.vertices[2].p);
+    }
 }

docs/architecture.md

@@ -529,7 +529,8 @@ with different vertex counts hold). Feather, Opacity and Expansion are ordinary
 - **GPU.** `filmcraft-gpu::GpuMask` evaluates the same coverage and mix in WGSL (compute), tested
   to agree with the CPU within 3·10⁻⁶. Layers with masks are CPU-rendered images in frame plans.
 - **Editing.** `masks.*` commands (add / remove / set / moveVertex / translate / addVertex /
-  removeVertex / toggleVertexSmooth / select / list); keyframe commands take `"mask": n`. The
+  removeVertex / toggleVertexSmooth / roundCorners / select / list; `masks.add` takes a
+  `cornerRadius` for a rounded rectangle); keyframe commands take `"mask": n`. The
   Program monitor overlay drags vertices, Bézier handles, the whole mask and the feather /
   expansion handles; drags merge into one undo step.
 - **Tracking** (`render::track`, `masks.track`): Shi–Tomasi features inside the mask, pyramidal

docs/gaps.md

@@ -228,11 +228,13 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement**
 ## G18. Effects depth: approximations and missing behaviour
 
 - **Missing:** Warp Stabilizer is 2-D (Premiere's is 3-D subspace), Morph Cut approximate, Lumetri
-  HDR-aware maths, multicam paging beyond 16 angles, redesigned 26.0 mask tools (rounded corners,
-  constrained lines).
+  HDR-aware maths, multicam paging beyond 16 angles, redesigned 26.0 mask tools (constrained
+  lines; an on-monitor corner-roundness handle).
 - **Have:** Optical Flow time interpolation synthesises motion-compensated in-between frames (dense
   coarse-to-fine Lucas–Kanade, CPU; #780); Transform shutter angle motion blur (#415); Ultra Key
-  Setting applied as a parameter preset (#458).
+  Setting applied as a parameter preset (#458); rectangle and polygon masks round their corners
+  with circular fillets (`masks.add` `cornerRadius`, `masks.roundCorners`, every path keyframe;
+  #813).
 - **Estimate:** 60–110 h.
 
 ## G19. Speech to text off in release builds