Quick Mask: ⌘I inverts the whole mask, unstored black included (#6095) (#6108)

* Quick Mask: ⌘I inverts the whole mask, unstored black included (#6095)

Image › Adjustments on an alpha channel or the Quick Mask ran on the surface's
stored content only. The black outside a selection-made Quick Mask (or a new
black alpha channel) is the surface default, never stored, so Invert left it
black: the whole mask turned red and leaving Quick Mask selected nothing.
Channel targets now go through `pixels::adjust_mask`, as layer masks do, which
adjusts the default pixel too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Quick Mask: test ⌘I with Color Indicates Selected Areas too (#6095)

A rectangle comes back as everything but the rectangle, and with nothing
selected ⌘I selects everything. Both failed on main.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/adjust_cmds/tests.rs

@@ -661,3 +661,53 @@ fn replace_color_localized_clusters_only_change_the_colour_near_the_sample() {
     let err = s.execute("image.adjustments.replaceColor", json!({"points": [[1]]})).unwrap_err();
     assert!(err.to_string().contains("bad point"), "{err}");
 }
+
+/// #6095: ⌘I in Quick Mask inverts the whole mask, not only where it was white, so leaving Quick
+/// Mask selects everything but the original rectangle, whichever way the colour reads. With
+/// Color Indicates Selected Areas and nothing selected, it selects everything. A black alpha
+/// channel inverts to white.
+#[test]
+fn invert_turns_the_black_of_quick_mask_and_alpha_channels_white() {
+    for depth in DEPTHS {
+        let mut s = session(depth, "rgb");
+        let pixels = rgba(&s, 2, 2);
+        s.execute("select.rect", json!({"x": 10, "y": 8, "width": 20, "height": 10})).unwrap();
+        s.execute("select.editInQuickMaskMode", json!({"on": true})).unwrap();
+        s.execute("image.adjustments.invert", json!({"target": "quickMask"})).unwrap();
+        let q = &doc(&s).quick_mask.as_ref().unwrap().surface;
+        let (inside, outside) = (q.sample_channel(15, 12, 0), q.sample_channel(2, 2, 0));
+        assert!(inside < 0.01 && outside > 0.99, "{depth}-bit Quick Mask inverted: inside {inside}, outside {outside}");
+        s.execute("select.editInQuickMaskMode", json!({"on": false})).unwrap();
+        let sel = doc(&s).selection.as_ref().expect("an inverse selection after Quick Mask");
+        let (inside, outside) = (sel.sample_channel(15, 12, 0), sel.sample_channel(2, 2, 0));
+        assert!(inside < 0.01 && outside > 0.99, "{depth}-bit selection: inside {inside}, outside {outside}");
+        assert_eq!(rgba(&s, 2, 2), pixels, "{depth}-bit: the layer is untouched");
+
+        s.execute("select.deselect", json!({})).unwrap();
+        s.execute("channel.new", json!({"name": "Black", "fill": "black"})).unwrap();
+        s.execute("image.adjustments.invert", json!({"target": {"channel": 0}})).unwrap();
+        let a = doc(&s).channels[0].surface.sample_channel(40, 30, 0);
+        assert!(a > 0.99, "{depth}-bit black alpha channel inverts to white ({a})");
+        s.undo();
+        assert!(doc(&s).channels[0].surface.sample_channel(40, 30, 0) < 0.01, "{depth}-bit: undo restores the black");
+
+        // Color Indicates Selected Areas: a rectangle comes back inverted, and nothing selected
+        // comes back as everything selected.
+        for (start, inside, outside) in [("rectangle", 0.0, 1.0), ("nothing", 1.0, 1.0)] {
+            let mut s = session(depth, "rgb");
+            s.execute("channel.options", json!({"channel": "quickMask", "indicates": "selected"})).unwrap();
+            if start == "rectangle" {
+                s.execute("select.rect", json!({"x": 10, "y": 8, "width": 20, "height": 10})).unwrap();
+            }
+            s.execute("select.editInQuickMaskMode", json!({"on": true})).unwrap();
+            s.execute("image.adjustments.invert", json!({"target": "quickMask"})).unwrap();
+            s.execute("select.editInQuickMaskMode", json!({"on": false})).unwrap();
+            let sel = doc(&s).selection.as_ref().unwrap_or_else(|| panic!("{depth}-bit Selected Areas, {start}: a selection after Quick Mask"));
+            let got = (sel.sample_channel(15, 12, 0), sel.sample_channel(2, 2, 0));
+            assert!(
+                (got.0 - inside).abs() < 0.01 && (got.1 - outside).abs() < 0.01,
+                "{depth}-bit Selected Areas, {start}: selection (inside, outside) {got:?}"
+            );
+        }
+    }
+}

crates/engine/src/commands.rs

@@ -381,12 +381,12 @@ fn destructive_adjust(s: &mut Session, label: &str, adj: Adjustment, p: &Value)
         });
     }
     if crate::channel_cmds::is_channel_target(p) {
-        // Alpha channel / Quick Mask target: the adjustment runs on the grayscale channel.
+        // Alpha channel / Quick Mask target: the adjustment runs on the grayscale channel as on
+        // a layer mask, its unstored black included, so ⌘I in Quick Mask inverts it all (#6095).
         return s.edit(label, |doc, _| {
             let sel = doc.selection.clone();
             if let Some(surf) = crate::channel_cmds::channel_surface_for_filter(doc, None, p)? {
-                pixels::adjust_surface(surf, &adj, sel.as_ref(), ColorMode::Grayscale);
-                surf.prune();
+                pixels::adjust_mask(surf, &adj, sel.as_ref());
             }
             Ok(Value::Null)
         });