Transparency: isolation mode and opacity mask editing can't be entered from each other

Enter Isolation Mode worked while an opacity mask was edited, and Edit
Opacity Mask and View Opacity Mask worked in isolation mode, so each
mode replaced the other's isolated art. In the reference app, isolation
mode can't be entered while a mask is edited, and mask editing can't be
entered in isolation mode.

object.isolate and perspective.editText, which shows type in isolation
mode, are now off while an opacity mask is edited, and a call returns
an error that ends "stop editing the opacity mask first". In isolation
mode transparency.editOpacityMask and transparency.viewOpacityMask are
off, and transparency.makeOpacityMask refuses one target, an object or
a layer, which it would take into mask editing; a call returns an error
that ends "exit isolation mode first". Make Opacity Mask with two
objects still masks one with the other. Pattern editing mode does not
count as isolation mode for these commands.

Object > Perspective > Edit Text, the canvas context menu and the
Transparency panel's menu and Make Mask button follow the commands'
enablement. A click or Alt-click on the mask thumbnail in isolation
mode shows the error and does not edit the mask. While a mask is
edited, the Contextual Task Bar leaves out Isolate, and the Properties
panel's Quick Actions leave out Isolate Group.

Docs: docs/mcp.md, docs/target-app-parity.md.

crates/engine/src/cmd/distortcmds.rs

@@ -198,8 +198,8 @@ pub fn specs() -> Vec<CommandSpec> {
             "Edit Text",
             [],
             None,
-            "{id? (default: the selected type)} Object › Perspective › Edit Text: show type in perspective flat where it is drawn, in isolation mode, to edit it (text.* commands, the Type tool); exiting isolation (object.exitIsolation, Esc) projects it again → {id}",
-            has_doc,
+            "{id? (default: the selected type)} Object › Perspective › Edit Text: show type in perspective flat where it is drawn, in isolation mode, to edit it (text.* commands, the Type tool); exiting isolation (object.exitIsolation, Esc) projects it again; refused while an opacity mask is edited → {id}",
+            super::maskedit::outside_mask_editing,
             edit_text
         ),
     ]

crates/engine/src/cmd/maskedit.rs

@@ -5,7 +5,9 @@
 //! object updates live; the renderer doesn't paint the editing layer (the mask is seen through its
 //! effect, as in the reference app). Leaving the mode removes the layer, and saving or exporting
 //! never writes it ([`Document::without_edit_modes`]). Meanwhile the Transparency panel's commands
-//! act on the masked object (see `opacitymask::transparency_targets`).
+//! act on the masked object (see `opacitymask::transparency_targets`). Mask editing is not entered
+//! in isolation mode ([`outside_isolation_mode`]), and isolation mode is not entered while a mask is
+//! edited ([`outside_mask_editing`]).
 
 use std::sync::Arc;
 
@@ -27,8 +29,8 @@ pub fn specs() -> Vec<CommandSpec> {
             "Edit Opacity Mask",
             ["Window", "Transparency"],
             None,
-            "{id?} edit the mask art of `id` (default: the first selected object with a mask) in place, as clicking the mask thumbnail in the Transparency panel does → {layer}",
-            has_doc,
+            "{id?} edit the mask art of `id` (default: the first selected object with a mask) in place, as clicking the mask thumbnail in the Transparency panel does; refused in isolation mode → {layer}",
+            outside_isolation_mode,
             enter
         ),
         cmd!(
@@ -45,13 +47,33 @@ pub fn specs() -> Vec<CommandSpec> {
             "View Opacity Mask",
             ["Window", "Transparency"],
             None,
-            "{on?: bool (default: toggle), id?} show only the opacity mask of `id` (default: the object whose mask is edited, else the first selected object with a mask) on the canvas, as greyscale coverage, and edit it, as Alt-clicking the mask thumbnail does; off shows the artwork again (still editing) → {on, id}",
-            has_doc,
+            "{on?: bool (default: toggle), id?} show only the opacity mask of `id` (default: the object whose mask is edited, else the first selected object with a mask) on the canvas, as greyscale coverage, and edit it, as Alt-clicking the mask thumbnail does; off shows the artwork again (still editing); refused in isolation mode → {on, id}",
+            outside_isolation_mode,
             view_mask
         ),
     ]
 }
 
+/// Is the document in isolation mode? Pattern editing mode and opacity mask editing isolate their
+/// art too, and do not count as isolation mode here.
+pub(crate) fn in_isolation_mode(st: &crate::DocState) -> bool {
+    st.isolation.is_some() && st.doc.pattern_edit.is_none() && st.doc.mask_edit.is_none()
+}
+
+/// A document is open and it is not in isolation mode ([`in_isolation_mode`]): opacity mask
+/// editing is not entered in isolation mode.
+pub(crate) fn outside_isolation_mode(s: &Session) -> std::result::Result<(), String> {
+    let st = s.active().ok_or("no document open")?;
+    if in_isolation_mode(st) { Err("exit isolation mode first".into()) } else { Ok(()) }
+}
+
+/// A document is open and no opacity mask is being edited: isolation mode is not entered while
+/// an opacity mask is edited.
+pub(crate) fn outside_mask_editing(s: &Session) -> std::result::Result<(), String> {
+    let st = s.active().ok_or("no document open")?;
+    if st.doc.mask_edit.is_some() { Err("stop editing the opacity mask first".into()) } else { Ok(()) }
+}
+
 /// Mask art made of `children`: a plain group, so the mask keeps one art node. Editing spreads it
 /// back out onto the editing layer.
 pub(crate) fn mask_group(children: Vec<Arc<Node>>) -> Node {

crates/engine/src/cmd/object/commands.rs

@@ -240,8 +240,8 @@ pub fn specs() -> Vec<CommandSpec> {
             "Enter Isolation Mode",
             [],
             None,
-            "{id?} isolate a group, a layer or a single object (default: the first selected object); type, a hidden or locked object, an object in a hidden or locked group or layer, and a guide path while Lock Guides is on are refused",
-            has_doc,
+            "{id?} isolate a group, a layer or a single object (default: the first selected object); type, a hidden or locked object, an object in a hidden or locked group or layer, and a guide path while Lock Guides is on are refused, and so is every object while an opacity mask is edited",
+            crate::cmd::maskedit::outside_mask_editing,
             isolate
         ),
         cmd!("object.exitIsolation", "Exit Isolation Mode", [], None, "{}", has_doc, exit_isolation),

crates/engine/src/cmd/opacitymask.rs

@@ -24,8 +24,8 @@ pub fn specs() -> Vec<CommandSpec> {
             "Make Opacity Mask",
             ["Window", "Transparency"],
             None,
-            "{ids?, clip?, invert?} the topmost of the selected objects (or of `ids`) becomes the mask of the others (grouped if several). One object gets an empty mask and enters mask editing: draw the mask, then transparency.stopEditingOpacityMask → {id, editing}",
-            has_doc,
+            "{ids?, clip?, invert?} the topmost of the selected objects (or of `ids`) becomes the mask of the others (grouped if several). One object gets an empty mask and enters mask editing: draw the mask, then transparency.stopEditingOpacityMask; one target (an object or a layer) is refused in isolation mode → {id, editing}",
+            can_make,
             make
         ),
         cmd!(
@@ -162,6 +162,20 @@ fn masked_targets(s: &Session, p: &Value) -> Result<Vec<NodeId>> {
     Ok(ids)
 }
 
+/// Make Opacity Mask: a document is open, and one selected object, which [`make`] gives an empty
+/// mask and takes into mask editing, is not in isolation mode
+/// ([`super::maskedit::outside_isolation_mode`]). The selection is one object when it has one
+/// subject root, which `make` masks without `ids`, and at most one outermost object, which it
+/// masks with `ids` (a call's `ids` stand in for the selection here). `make` itself refuses any
+/// single target in isolation mode, a layer included.
+fn can_make(s: &Session) -> std::result::Result<(), String> {
+    has_doc(s)?;
+    let Some(st) = s.active() else { return Ok(()) };
+    let roots = super::appearance::subject_roots(s).map_or(0, |r| r.len());
+    let outermost = super::edit::outermost(&st.doc, st.selection.in_paint_order(&st.doc)).len();
+    if roots == 1 && outermost <= 1 { super::maskedit::outside_isolation_mode(s) } else { Ok(()) }
+}
+
 fn make(s: &mut Session, p: &Value) -> Result<Value> {
     const C: &str = "transparency.makeOpacityMask";
     let d = &s.doc()?.doc;
@@ -178,6 +192,10 @@ fn make(s: &mut Session, p: &Value) -> Result<Value> {
         None => super::appearance::subject_roots(s)?,
     };
     let Some((&top, below)) = ids.split_last() else { return Err(bad(C, "select the art and, on top of it, the mask object")) };
+    // One target gets an empty mask and enters mask editing, which isolation mode does not allow.
+    if below.is_empty() && super::maskedit::in_isolation_mode(s.doc()?) {
+        return Err(bad(C, "exit isolation mode first"));
+    }
     let clip = bool_or(p, "clip", !s.menu.new_masks_unclipped);
     let invert = bool_or(p, "invert", s.menu.new_masks_inverted);
     // Several objects: the top one is the mask. One: an empty mask, drawn in mask editing.

crates/engine/src/tests/opacitymask.rs

@@ -1,6 +1,7 @@
 //! Opacity-mask editing and the Transparency panel's command reach (M3.25): commands act on the
 //! masked object while its mask is edited, accept ids without a selection, take opacity in percent,
-//! report mixed values, and saves and exports never carry the editing layer.
+//! report mixed values, and saves and exports never carry the editing layer. Mask editing is not
+//! entered in isolation mode, and isolation mode is not entered while a mask is edited.
 
 use serde_json::{Value, json};
 use vectorcraft_geom::Affine;
@@ -228,6 +229,85 @@ fn one_object_gets_an_empty_mask_drawn_in_editing_mode() {
     assert_eq!((s.doc().unwrap().isolation, s.doc().unwrap().active_layer), (Some(layer), Some(layer)));
 }
 
+/// The command `cmd` with `p` is off, for `why`.
+fn off(s: &mut Session, cmd: &str, p: Value, why: &str) {
+    let err = s.execute(cmd, &p).unwrap_err();
+    assert!(matches!(&err, EngineError::Disabled(id, w) if id == cmd && w == why), "{cmd} {p}: {err}");
+}
+
+/// Isolation mode is not entered while an opacity mask is edited: Enter Isolation Mode is off for
+/// a group in the mask art and for one outside it, and mask editing goes on. After Stop Editing
+/// Opacity Mask it isolates again.
+#[test]
+fn isolation_mode_is_not_entered_while_an_opacity_mask_is_edited() {
+    let mut s = session();
+    let group = |s: &mut Session, x: f64| {
+        let (a, b) = (rect(s, x, 20.0), rect(s, x + 30.0, 20.0));
+        s.execute("select.set", &json!({"ids": [a.0, b.0]})).unwrap();
+        NodeId(s.execute("object.group", &json!({})).unwrap()["id"].as_u64().unwrap())
+    };
+    let outside = group(&mut s, 220.0);
+    editing_mask(&mut s);
+    let layer = s.doc().unwrap().isolation;
+    let inside = group(&mut s, 0.0);
+    for id in [inside, outside] {
+        off(&mut s, "object.isolate", json!({"id": id.0}), "stop editing the opacity mask first");
+        assert_eq!(s.doc().unwrap().isolation, layer);
+        assert!(doc(&s).mask_edit.is_some(), "still editing");
+    }
+    s.execute("select.set", &json!({"ids": [inside.0]})).unwrap();
+    off(&mut s, "object.isolate", json!({}), "stop editing the opacity mask first");
+    s.execute("transparency.stopEditingOpacityMask", &json!({})).unwrap();
+    s.execute("object.isolate", &json!({"id": outside.0})).unwrap();
+    assert_eq!(s.doc().unwrap().isolation, Some(outside));
+}
+
+/// Opacity mask editing is not entered in isolation mode: Edit Opacity Mask and View Opacity Mask
+/// are off, and so is Make Opacity Mask for one object, which would edit its new empty mask. Make
+/// Opacity Mask with two objects still masks one with the other. After Exit Isolation Mode the mask
+/// is edited again.
+#[test]
+fn opacity_mask_editing_is_not_entered_in_isolation_mode() {
+    let mut s = session();
+    let (obj, m) = (rect(&mut s, 0.0, 200.0), rect(&mut s, 0.0, 100.0));
+    s.execute("transparency.makeOpacityMask", &json!({"ids": [obj.0, m.0]})).unwrap();
+    let other = rect(&mut s, 220.0, 30.0);
+    s.execute("select.set", &json!({"ids": [obj.0, other.0]})).unwrap();
+    let g = NodeId(s.execute("object.group", &json!({})).unwrap()["id"].as_u64().unwrap());
+    s.execute("object.isolate", &json!({"id": g.0})).unwrap();
+    s.execute("select.set", &json!({"ids": [obj.0]})).unwrap();
+    let why = "exit isolation mode first";
+    off(&mut s, "transparency.editOpacityMask", json!({}), why);
+    off(&mut s, "transparency.editOpacityMask", json!({"id": obj.0}), why);
+    off(&mut s, "transparency.viewOpacityMask", json!({}), why);
+    off(&mut s, "transparency.viewOpacityMask", json!({"on": true, "id": obj.0}), why);
+    s.execute("select.set", &json!({"ids": [other.0]})).unwrap();
+    off(&mut s, "transparency.makeOpacityMask", json!({}), why);
+    off(&mut s, "transparency.makeOpacityMask", json!({"ids": [other.0]}), why);
+    // A layer given as the one target is refused too.
+    let layer = doc(&s).layers[0].id;
+    let err = s.execute("transparency.makeOpacityMask", &json!({"ids": [layer.0]})).unwrap_err();
+    assert!(err.to_string().ends_with(why), "{err}");
+    let st = s.doc().unwrap();
+    assert!(st.doc.mask_edit.is_none() && st.mask_view.is_none() && mask(&s, other).is_none());
+    assert_eq!(s.doc().unwrap().isolation, Some(g));
+    // Two objects: the top one masks the other, without mask editing.
+    let top = rect(&mut s, 220.0, 10.0);
+    let r = s.execute("transparency.makeOpacityMask", &json!({"ids": [other.0, top.0]})).unwrap();
+    assert_eq!(r, json!({"id": other.0, "editing": false}));
+    assert_eq!(s.doc().unwrap().isolation, Some(g));
+    // Two members of a compound path given in `ids` are two objects: one masks the other.
+    let (c1, c2) = (rect(&mut s, 250.0, 10.0), rect(&mut s, 265.0, 10.0));
+    s.execute("select.set", &json!({"ids": [c1.0, c2.0]})).unwrap();
+    s.execute("object.compoundPath.make", &json!({})).unwrap();
+    let r = s.execute("transparency.makeOpacityMask", &json!({"ids": [c1.0, c2.0]})).unwrap();
+    assert_eq!(r["editing"], json!(false));
+    assert!(doc(&s).mask_edit.is_none());
+    s.execute("object.exitIsolation", &json!({})).unwrap();
+    s.execute("transparency.editOpacityMask", &json!({"id": obj.0})).unwrap();
+    assert_eq!(doc(&s).mask_edit.map(|e| e.object), Some(obj));
+}
+
 #[test]
 fn mask_and_transparency_commands_take_ids_without_a_selection() {
     let mut s = session();

crates/engine/src/tests/persp_text.rs

@@ -141,6 +141,23 @@ fn edit_text_shows_it_flat_in_isolation_then_projects_it_again() {
     assert!(s2.execute("perspective.editText", &json!({"id": rect.0})).is_err());
 }
 
+/// Edit Text, which shows the type in isolation mode, is off while an opacity mask is edited.
+#[test]
+fn edit_text_is_off_while_an_opacity_mask_is_edited() {
+    let (mut s, rect) = persp_session();
+    let id = persp_text(&mut s);
+    s.execute("transparency.makeOpacityMask", &json!({"ids": [rect.0]})).unwrap();
+    let layer = s.doc().unwrap().isolation;
+    assert!(layer.is_some(), "editing the new empty mask");
+    let err = s.execute("perspective.editText", &json!({"id": id.0})).unwrap_err();
+    assert!(matches!(&err, EngineError::Disabled(_, why) if why == "stop editing the opacity mask first"), "{err}");
+    assert_eq!(s.doc().unwrap().isolation, layer);
+    assert!(node(&s, id).projection().is_some(), "still projected");
+    s.execute("transparency.stopEditingOpacityMask", &json!({})).unwrap();
+    s.execute("perspective.editText", &json!({"id": id.0})).unwrap();
+    assert_eq!(s.doc().unwrap().isolation, Some(id));
+}
+
 /// Edit Text ending while a background save writes the file leaves the document saved once the
 /// file is written: the flat view is never saved. Saved while editing and edited after: undoing
 /// the edit once Edit Text has ended is back at what the file holds.