Effects: Fractal and Turbulent Noise stay inside the layer's alpha (#939)

Masks are applied before effects, but the noise was composited with the
Opacity alpha alone, so a masked solid filled the whole layer. The result
is now multiplied by the original alpha on the CPU and on both GPU paths
(kernels.wgsl for the default settings, fx_noise.wgsl for the rest).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

crates/effects/src/noise3.rs

@@ -429,7 +429,8 @@ pub fn overflow(v: f32, mode: u32) -> f32 {
 }
 
 /// Composite grey noise `val` over the original pixel by Blending Mode, Opacity and (ours) Blend
-/// With Original.
+/// With Original. The noise stays inside the layer's alpha, so a masked solid shows it only within
+/// its masks (#939).
 pub(crate) fn composite(orig: [f32; 4], val: f32, opacity: f32, mode: u32, blend_orig: f32) -> [f32; 4] {
     let src = [val * opacity, val * opacity, val * opacity, opacity];
     let out = match mode {
@@ -440,7 +441,7 @@ pub(crate) fn composite(orig: [f32; 4], val: f32, opacity: f32, mode: u32, blend
         }
     };
     let k = 1.0 - blend_orig;
-    [0, 1, 2, 3].map(|c| orig[c] * blend_orig + out[c] * k)
+    [0, 1, 2, 3].map(|c| orig[c] * blend_orig + out[c] * orig[3] * k)
 }
 
 fn render(ctx: &EffectCtx, mut b: Buf, turbulent: bool) -> Buf {
@@ -574,6 +575,28 @@ mod tests {
         assert!(mul.data.iter().any(|p| p[2] > p[0] + 0.01));
     }
 
+    #[test]
+    fn noise_stays_inside_the_layer_alpha() {
+        // #939: a masked solid (transparent outside the mask, a soft edge) filled the whole layer.
+        let mut img = Image::filled(40, 24, [0.2, 0.4, 0.6, 1.0]);
+        for (i, p) in img.data.iter_mut().enumerate() {
+            let a = match i % 40 {
+                0..10 => 0.0,
+                10..20 => 0.5,
+                _ => 1.0,
+            };
+            *p = p.map(|c| c * a);
+        }
+        for mode in [BLEND_NONE, BLEND_NORMAL, 3] {
+            let out = run_fx("ec.noise.fractal", &[("blendingMode", Value::Enum(mode))], img.clone(), 0.0, EffectEnv::default()).img;
+            for (o, p) in img.data.iter().zip(&out.data) {
+                assert!((o[3] - p[3]).abs() < 1e-6, "mode {mode}: alpha {} became {}", o[3], p[3]);
+                assert!(p[..3].iter().all(|&c| c <= p[3] + 1e-6), "mode {mode}: premultiplied colour above alpha");
+            }
+            assert!(out.data.iter().any(|p| p[3] == 1.0 && (p[0] - 0.2).abs() > 0.01), "mode {mode}: noise drawn inside the mask");
+        }
+    }
+
     #[test]
     fn sub_settings_and_fractal_types_change_the_result() {
         let base = run("ec.noise.fractal", &[]);

crates/gpu/src/shaders/fx_noise.wgsl

@@ -665,7 +665,7 @@ fn fxn_fractal(@builtin(global_invocation_id) gid: vec3<u32>) {
         r = blend_pixel(P.u[2].x, orig, s, 0.5);
     }
     let bo = P.f[1].w;
-    textureStore(out, p, orig * bo + r * (1.0 - bo));
+    textureStore(out, p, orig * bo + r * orig.a * (1.0 - bo));
 }
 
 // ---------------------------------------------------------------- Noise Alpha / Noise HLS (noise.rs)

crates/gpu/src/shaders/kernels.wgsl

@@ -858,13 +858,13 @@ fn pointwise(@builtin(global_invocation_id) gid: vec3<u32>) {
                 val = 1.0 - val;
             }
             let op = P.f[2].z;
+            var fx = vec4<f32>(vec3<f32>(val) * op, op);
             if (P.u[1].z == 1u) {
-                // Normal: the noise over the layer at Opacity, then Blend With Original.
-                let over = vec4<f32>(vec3<f32>(val) * op, op) + px * (1.0 - op);
-                px = px * P.f[2].y + over * (1.0 - P.f[2].y);
-            } else {
-                px = put(px, vec3<f32>(val), op, P.f[2].y);
+                // Normal: the noise over the layer at Opacity.
+                fx = fx + px * (1.0 - op);
             }
+            // Inside the layer's alpha (its masks), then Blend With Original.
+            px = px * P.f[2].y + fx * px.w * (1.0 - P.f[2].y);
         }
         default: {}
     }