Merge #1917: #1812: stills at reduced resolution are resampled once, and their layers cached (@ded-achtung)

crates/media/src/pool.rs

@@ -45,6 +45,9 @@ struct Key {
     matte: [u32; 3],
     /// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded).
     size: (u32, u32),
+    /// A still at a reduced `size`: the footage's nominal size, which decides whether it is
+    /// resampled (see `Inner::still_at_size`); (0, 0) otherwise.
+    nominal: (u32, u32),
 }
 
 struct Entry {
@@ -678,7 +681,7 @@ impl Inner {
                 FloatCurve::Gamma24 => 2,
             } << 2);
         let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] };
-        let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) };
+        let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0), nominal: (0, 0) };
         match footage.kind {
             FootageKind::Sequence if !footage.sequence.is_empty() => {
                 let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count);
@@ -708,16 +711,20 @@ impl Inner {
         }
     }
 
-    /// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that
-    /// size directly (see [`FootageSource::frame_at_size`]).
+    /// The frame of `footage` at `t`, resampled to `size` (see [`FootageSource::frame_at_size`]):
+    /// a movie's made at that size directly, a still's resampled once and kept at that size.
     fn frame_at(self: &Arc<Self>, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>, float: FloatCurve) -> Result<Arc<Image>> {
         if !footage.has_video {
             return Err(MediaError::Unsupported(format!("{}: no video", footage.path)));
         }
         let mut loc = Self::locate(footage, t, float);
         let movie = loc.media_t.is_some();
-        if movie && let Some(size) = size {
+        // A still asked for at its own size is the unsized frame (one decode, one entry).
+        if let Some(size) = size.filter(|s| movie || *s != (footage.width, footage.height)) {
             loc.key.size = size;
+            if !movie {
+                loc.key.nominal = (footage.width, footage.height);
+            }
         }
         let mut c = lock(&self.cache);
         // sequential playback of a movie: read the next frame ahead
@@ -773,34 +780,62 @@ impl Inner {
     #[cfg(target_arch = "wasm32")]
     fn prefetch(self: &Arc<Self>, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>, _float: FloatCurve) {}
 
+    /// A still (image file or sequence frame) at `loc.key.size` when one is set: resampled as the
+    /// renderer would ([`FootageSource::frame_at_size`]), so the pool keeps the reduced frame and a
+    /// reduced-resolution render neither decodes nor resamples a large still again for every frame
+    /// (#1812: 21120-px plates at Quarter). Like the renderer's `footage_buf`, an image decoded
+    /// larger than the footage's nominal size (a smart object's pixels) is resampled too, and one
+    /// decoded smaller is not (the renderer scales the layer instead).
+    fn still_at_size(img: Image, loc: &Loc, footage: &Footage) -> Image {
+        let (w, h) = loc.key.size;
+        let smaller = img.width == 0 || footage.width as f64 / img.width as f64 > 1.0 + 1e-9;
+        if w == 0 || h == 0 || smaller || (w, h) == (img.width, img.height) {
+            return img;
+        }
+        effectcraft_raster::resample(&img, w, h)
+    }
+
     fn decode(&self, loc: &Loc, footage: &Footage) -> Result<Image> {
+        match loc.media_t {
+            None => self.decode_still(loc, footage).map(|img| Self::still_at_size(img, loc, footage)),
+            Some(_) => self.decode_media(loc, footage),
+        }
+    }
+
+    fn decode_still(&self, loc: &Loc, footage: &Footage) -> Result<Image> {
+        let op = AlphaOp::of(footage, loc.float);
+        let path = &loc.key.path;
+        if &**path == PLACEHOLDER {
+            return Ok(colour_bars(footage.width, footage.height));
+        }
+        let bytes = self.read(path)?;
+        if let Some(img) = crate::dpx::decode(path, &bytes)? {
+            return Ok(dynamic_to_image(&img, op));
+        }
+        if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? {
+            return Ok(img);
+        }
+        // (an OpenEXR compression the decoder lacks came out transparent or as a missing
+        // channel error)
+        if path.to_ascii_lowercase().ends_with(".exr")
+            && let Some(why) = crate::exr_channels::unsupported_compression(&bytes)
+        {
+            return Err(MediaError::Decode(format!("{path}: {why}")));
+        }
+        let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes));
+        let img = match loaded {
+            Ok(img) => img,
+            // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer.
+            Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?,
+        };
+        Ok(dynamic_to_image(&img, op))
+    }
+
+    fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result<Image> {
         let op = AlphaOp::of(footage, loc.float);
         let path = &loc.key.path;
         match loc.media_t {
-            None if &**path == PLACEHOLDER => Ok(colour_bars(footage.width, footage.height)),
-            None => {
-                let bytes = self.read(path)?;
-                if let Some(img) = crate::dpx::decode(path, &bytes)? {
-                    return Ok(dynamic_to_image(&img, op));
-                }
-                if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? {
-                    return Ok(img);
-                }
-                // (an OpenEXR compression the decoder lacks came out transparent or as a missing
-                // channel error)
-                if path.to_ascii_lowercase().ends_with(".exr")
-                    && let Some(why) = crate::exr_channels::unsupported_compression(&bytes)
-                {
-                    return Err(MediaError::Decode(format!("{path}: {why}")));
-                }
-                let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes));
-                let img = match loaded {
-                    Ok(img) => img,
-                    // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer.
-                    Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?,
-                };
-                Ok(dynamic_to_image(&img, op))
-            }
+            None => self.decode_still(loc, footage),
             Some(mt) if gif_anim::is_animated(footage) => {
                 let anim = self.gif(path)?;
                 let frame = anim.frame_at(Tick(mt.0)).ok_or_else(|| MediaError::Decode(format!("{path}: no frames")))?;
@@ -1045,7 +1080,7 @@ mod tests {
     fn lru_respects_budget_and_recycles() {
         let mut c = Lru::default();
         let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead
-        let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) };
+        let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0), nominal: (0, 0) };
         let budget = 3 * (4096 + 64);
         for i in 0..10 {
             c.insert(k(i), img(), budget);

crates/media/tests/it/footage.rs

@@ -584,6 +584,46 @@ fn sequence_gaps_show_placeholders() {
     let _ = std::fs::remove_dir_all(&dir);
 }
 
+/// #1812: a still asked for at a reduced size comes back as the renderer would resample it, and
+/// is kept at that size: later frames at that size neither decode nor resample it again (a
+/// parallax project's 21120-px plates were decoded and resampled for every frame at Quarter).
+#[test]
+fn stills_at_reduced_sizes_are_resampled_once() {
+    let dir = std::env::temp_dir().join(format!("ec-still-size-{}", std::process::id()));
+    std::fs::create_dir_all(&dir).expect("temp dir");
+    let p = dir.join("plate.png");
+    let px = |x: u32, y: u32| {
+        image::Rgba([(x * 7 % 256) as u8, (y * 13 % 256) as u8, ((x ^ y) % 256) as u8, if (x + y).is_multiple_of(5) { 0 } else { 255 - (x % 128) as u8 }])
+    };
+    image::RgbaImage::from_fn(613, 97, px).save(&p).expect("png");
+    let f = probe(&p).expect("probe");
+    let pool = MediaPool::new();
+    let full = pool.frame(ItemId(1), &f, Tick::ZERO).expect("frame");
+    // At its own size it is the unsized frame: no second decode, no second full-size copy.
+    let misses = pool.stats().misses;
+    let native = pool.frame_at_size(ItemId(1), &f, Tick::ZERO, f.width, f.height).expect("frame");
+    assert!(Arc::ptr_eq(&full, &native), "the native size reuses the unsized frame");
+    assert_eq!(pool.stats().misses, misses);
+    for (w, h) in [(306, 48), (153, 24), (77, 12)] {
+        let sized = pool.frame_at_size(ItemId(1), &f, Tick::ZERO, w, h).expect("frame");
+        assert_eq!(*sized, effectcraft_raster::resample(&full, w, h), "{w}×{h}: the renderer's resample");
+        let misses = pool.stats().misses;
+        let again = pool.frame_at_size(ItemId(1), &f, Tick(3 * TICKS_PER_SECOND), w, h).expect("frame");
+        assert!(Arc::ptr_eq(&sized, &again), "{w}×{h}: kept at that size");
+        assert_eq!(pool.stats().misses, misses, "{w}×{h}: not decoded again");
+    }
+    // Pixels larger than the nominal size (a smart object's) are resampled as the renderer does;
+    // smaller ones are left as they are (the renderer scales the layer up instead).
+    // One pool, one file, one requested size: the two decisions don't share a cache entry.
+    let larger = Footage { width: 306, height: 48, ..f.clone() };
+    let smaller = Footage { width: 1226, height: 194, ..f.clone() };
+    let sized = pool.frame_at_size(ItemId(2), &larger, Tick::ZERO, 153, 24).expect("frame");
+    assert_eq!(*sized, effectcraft_raster::resample(&full, 153, 24));
+    let kept = pool.frame_at_size(ItemId(3), &smaller, Tick::ZERO, 153, 24).expect("frame");
+    assert_eq!(*kept, *full);
+    let _ = std::fs::remove_dir_all(&dir);
+}
+
 #[test]
 fn png_with_alpha() {
     let Some(p) = alpha_png() else { return };

crates/render/src/cache.rs

@@ -22,7 +22,7 @@ use std::hash::{Hash, Hasher};
 use std::sync::{Arc, Mutex};
 
 use effectcraft_effects::Buf;
-use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup};
+use effectcraft_project::{FootageKind, ItemKind, Layer, LayerSource, Node, PropGroup};
 
 use crate::eval::EvalCtx;
 
@@ -481,7 +481,7 @@ fn hash_values(h: &mut KeyHasher, ctx: &EvalCtx, layer: &Layer, g: &PropGroup) {
 }
 
 /// Cache key for the processed (source → masks → effects) buffer of `layer` at the context
-/// time, or `None` when the layer must not be cached (footage, precomps, cameras, adjustment
+/// time, or `None` when the layer must not be cached (movie footage, precomps, cameras, adjustment
 /// layers…).
 pub fn layer_key(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, blur: bool) -> Option<u64> {
     // Time effects see property values at other times. Keyframes are part of the hashed
@@ -514,6 +514,34 @@ fn reads_other_times(layer: &Layer) -> bool {
         })
 }
 
+/// A still image (no proxy): its layer is cached like a text or shape layer, so the GPU
+/// compositor uploads it once, not every frame (#1812: four 21120 px plates re-uploaded by every
+/// preview worker ran the GPU out of memory). Movies and sequences stay in the media pool only.
+fn still_footage(ctx: &EvalCtx, item: effectcraft_project::ItemId) -> bool {
+    ctx.project.item(item).is_some_and(|it| it.proxy.is_none() && matches!(&it.kind, ItemKind::Footage(f) if f.kind == FootageKind::Still))
+}
+
+/// Does the layer have an animated mask that motion blur may draw at other times (Layer ▸ Mask ▸
+/// Motion Blur On, or Same As Layer with the layer blurred)?
+fn mask_motion_blur(ctx: &EvalCtx, layer: &Layer, blur: bool) -> bool {
+    use effectcraft_project::{GroupKind, MaskMotionBlur};
+    fn animated(g: &PropGroup) -> bool {
+        g.children.iter().any(|c| match c {
+            Node::Prop(p) => !p.keys.is_empty() || p.has_expression(),
+            Node::Group(sub) => animated(sub),
+        })
+    }
+    ctx.comp.enable_motion_blur
+        && layer.masks().is_some_and(|m| {
+            m.groups().any(|g| match g.kind {
+                GroupKind::Mask { motion_blur, .. } => {
+                    (motion_blur == MaskMotionBlur::On || (motion_blur == MaskMotionBlur::SameAsLayer && blur)) && animated(g)
+                }
+                _ => false,
+            })
+        })
+}
+
 fn has_expression(g: &PropGroup) -> bool {
     g.children.iter().any(|c| match c {
         Node::Prop(p) => p.has_expression(),
@@ -552,14 +580,17 @@ fn key_any(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, blur: bool, fo
             hash_debug(&mut h, s);
         }
         LayerSource::Text | LayerSource::Shape => {}
-        LayerSource::Footage { item } if footage => {
+        LayerSource::Footage { item } if footage || still_footage(ctx, *item) => {
             let it = ctx.project.item(*item)?;
             let ItemKind::Footage(f) = &it.kind else { return None };
             hash_debug(&mut h, f);
             // A proxy (and its Use Proxy switch) changes the pixels.
             hash_debug(&mut h, &it.proxy);
             item.hash(&mut h);
-            ctx.source_time(layer).0.hash(&mut h);
+            // A still is the same at every time (unless a proxy, which may be a movie, stands in).
+            if f.kind != FootageKind::Still || it.proxy.is_some() {
+                ctx.source_time(layer).0.hash(&mut h);
+            }
             // Frame blending mixes neighbouring source frames.
             ctx.comp.enable_frame_blending.hash(&mut h);
         }
@@ -625,6 +656,11 @@ fn key_any(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, blur: bool, fo
             effectcraft_effects::paint::cache_key(g, lt).0.hash(&mut h);
         }
     }
+    // Mask motion blur draws animated masks at the shutter's other times too: their values at
+    // this time don't tell two frames apart (a hold before a move).
+    if mask_motion_blur(ctx, layer, blur) {
+        layer.layer_time(ctx.time).0.hash(&mut h);
+    }
     if time_dependent(layer, effects != Some(0)) {
         layer.layer_time(ctx.time).0.hash(&mut h);
         // Effects also see the layer's time mapping.

crates/render/src/tests_footage_size.rs

@@ -81,3 +81,28 @@ fn reduced_renders_take_footage_resampled_by_the_source() {
     Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO);
     assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty());
 }
+
+/// #1812: a still's layer is kept in the layer cache across times (its texture then uploads to
+/// the GPU once, not every frame); a movie's is fetched again at every time.
+#[test]
+fn still_layers_are_cached_across_times_and_movies_are_not() {
+    for (kind, fetches) in [(FootageKind::Still, 1), (FootageKind::Video, 3)] {
+        let (mut p, cid) = project(BitDepth::Bpc8, false);
+        let Some(LayerSource::Footage { item }) = p.comp(cid).map(|c| c.layers[0].source.clone()) else { panic!("footage layer") };
+        if let Some(ItemKind::Footage(f)) = p.item_mut(item).map(|it| &mut it.kind) {
+            f.kind = kind;
+        }
+        let cache = crate::LayerCache::default();
+        let src = Clip { sizes: Some(Mutex::default()) };
+        let opts = RenderOpts { scale: 0.5, ..Default::default() };
+        for t in [0.0, 0.25, 0.5] {
+            let t = Tick::from_seconds_f64(t);
+            let mut r = Renderer::new(&p, &src, opts);
+            r.cache = Some(&cache);
+            let frame = r.comp_frame(cid, t);
+            assert_eq!(frame, Renderer::new(&p, &src, opts).comp_frame(cid, t), "{kind:?} at {t:?}");
+        }
+        // (each uncached render above fetches once more)
+        assert_eq!(src.sizes.unwrap().into_inner().unwrap().len(), fetches + 3, "{kind:?}");
+    }
+}

crates/render/src/tests_mask_blur.rs

@@ -41,6 +41,31 @@ fn alpha_at_25(p: &Project, cid: ItemId, motion_blur: bool) -> f32 {
     Renderer::new(p, &NoFootage, RenderOpts { motion_blur, ..Default::default() }).comp_frame(cid, half).get(25, 50)[3]
 }
 
+/// A mask that holds from frame 0 to 2, then moves: frames 1 and 2 have the same mask value, but at
+/// frame 2 the shutter already sees it move. The layer cache must not serve frame 1's layer there.
+#[test]
+fn mask_motion_blur_keys_the_layer_cache_by_time() {
+    let (mut p, cid) = project(true);
+    let rect = |x: f64| Value::Path(ShapePath::rect([x, 50.0], 20.0, 100.0));
+    let l = &mut p.comp_mut(cid).unwrap().layers[0];
+    let Some(Node::Group(m)) = l.props.sub_mut("masks").unwrap().children.first_mut() else { panic!("mask") };
+    m.get_mut("path").unwrap().keys = vec![
+        Keyframe::new(Tick::ZERO, rect(20.0)),
+        Keyframe::new(FrameRate::FPS_30.tick_of(2), rect(20.0)),
+        Keyframe::new(FrameRate::FPS_30.tick_of(3), rect(60.0)),
+    ];
+    let cache = crate::LayerCache::default();
+    let render = |t: Tick, cached: bool| {
+        let mut r = Renderer::new(&p, &NoFootage, RenderOpts { motion_blur: true, ..Default::default() });
+        r.cache = cached.then_some(&cache);
+        r.comp_frame(cid, t)
+    };
+    let (f1, f2) = (FrameRate::FPS_30.tick_of(1), FrameRate::FPS_30.tick_of(2));
+    assert_ne!(render(f1, false), render(f2, false), "the shutter at frame 2 sees the move");
+    render(f1, true);
+    assert_eq!(render(f2, true), render(f2, false));
+}
+
 #[test]
 fn mask_motion_blur_follows_the_render_and_the_comp() {
     let (p, cid) = project(true);