mirror of
https://github.com/storytold/deckcraft.git
synced 2026-10-28 04:27:06 -06:00
feat(pictures): animated GIFs play, with pause (media.gifPlay) (#39)
* feat(pictures): animated GIFs play, with pause (media.gifPlay) GIF pictures used to show only their first frame. The renderer now decodes every frame (cached, capped at 256 MB and 1024 px a side) and draws the frame for each GIF's play time, passed in RenderOpts::gif_times; exports and thumbnails pass none and keep frame 1. The editor canvas and the slide show redraw only when a GIF reaches its next frame. A corner button on hover (or selection) pauses and resumes a GIF from the same frame, through the new media.gifPlay command. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat(pictures): animated GIFs play, with pause (media.gifPlay) GIF pictures used to show only their first frame. The renderer now decodes every frame (cached, capped at 256 MB and 1024 px a side) and draws the frame for each GIF's play time, passed in RenderOpts::gif_times; exports and thumbnails pass none and keep frame 1. The editor canvas and the slide show redraw only when a GIF reaches its next frame. A corner button on hover (or selection) pauses and resumes a GIF from the same frame, through the new media.gifPlay command. Also adds my entry to contributors/people.toml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Animated GIFs: cap frames (1000), canvas (4096²), per-GIF (64 MB) and cache (192 MB) memory A few MB of hostile GIF could decode to millions of tiny frames or a 100 Mpx canvas per frame, and eight cached GIFs could hold 2 GB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
Dmytro Smaha
co-authored by
Claude Opus 5.5
Brandon Thomas
Claude Opus 5.5
Brandon Thomas
parent
e8c4c60884
commit
3985d768e6
+140
-12
@@ -21,6 +21,41 @@ struct Cache {
|
||||
bytes: usize,
|
||||
clock: u64,
|
||||
bad: HashMap<usize, Arc<Vec<u8>>>,
|
||||
/// GIFs by bytes address and adjustments (`None`: not animated); the bytes stay alive with them.
|
||||
anims: HashMap<(usize, u64), (Arc<Vec<u8>>, Option<Arc<Anim>>)>,
|
||||
}
|
||||
|
||||
/// Longest side, total decoded size and frame count kept for one animated GIF; the largest canvas
|
||||
/// decoded at all; and the decoded size of all cached GIFs together. Hostile GIFs (millions of
|
||||
/// tiny frames, a huge canvas) stop at these instead of exhausting memory.
|
||||
const GIF_SIDE: u32 = 1024;
|
||||
const GIF_BUDGET: usize = 64 << 20;
|
||||
const GIF_MAX_FRAMES: usize = 1000;
|
||||
const GIF_MAX_CANVAS: u64 = 4096 * 4096;
|
||||
const GIF_CACHE_BUDGET: usize = 192 << 20;
|
||||
|
||||
/// The frames of an animated GIF and when each one ends, in seconds from the start of a loop.
|
||||
struct Anim {
|
||||
frames: Vec<Arc<Pixmap>>,
|
||||
ends: Vec<f64>,
|
||||
}
|
||||
|
||||
impl Anim {
|
||||
/// Decoded bytes held by the frames.
|
||||
fn bytes(&self) -> usize {
|
||||
self.frames.iter().map(|f| f.width() as usize * f.height() as usize * 4).sum()
|
||||
}
|
||||
|
||||
/// The frame showing `t` seconds into the (looping) animation and the seconds until the next.
|
||||
fn at(&self, t: f64) -> (usize, f64) {
|
||||
let total = self.ends.last().copied().unwrap_or(0.0);
|
||||
if total <= 0.0 || !t.is_finite() {
|
||||
return (0, 1.0);
|
||||
}
|
||||
let local = t.max(0.0) % total;
|
||||
let i = self.ends.partition_point(|e| *e <= local).min(self.ends.len().saturating_sub(1));
|
||||
(i, (self.ends.get(i).copied().unwrap_or(total) - local).max(0.001))
|
||||
}
|
||||
}
|
||||
|
||||
static CACHE: Mutex<Option<Cache>> = Mutex::new(None);
|
||||
@@ -94,18 +129,7 @@ pub fn decode_for(bytes: &Arc<Vec<u8>>, want: f64, adj: &PictureAdjust) -> Optio
|
||||
}
|
||||
apply_adjust(&mut img, adj);
|
||||
let (w, h) = img.dimensions();
|
||||
if w > u16::MAX as u32 || h > u16::MAX as u32 {
|
||||
return None;
|
||||
}
|
||||
let data: Vec<vello_cpu::color::PremulRgba8> = img
|
||||
.pixels()
|
||||
.map(|p| {
|
||||
let a = p[3] as u16;
|
||||
let m = |c: u8| ((c as u16 * a + 127) / 255) as u8;
|
||||
vello_cpu::color::PremulRgba8 { r: m(p[0]), g: m(p[1]), b: m(p[2]), a: p[3] }
|
||||
})
|
||||
.collect();
|
||||
let pm = Arc::new(Pixmap::from_parts(data, w as u16, h as u16));
|
||||
let pm = pixmap(&img)?;
|
||||
with(|c| {
|
||||
c.clock += 1;
|
||||
let n = w as usize * h as usize * 4;
|
||||
@@ -121,6 +145,89 @@ pub fn decode_for(bytes: &Arc<Vec<u8>>, want: f64, adj: &PictureAdjust) -> Optio
|
||||
Some(pm)
|
||||
}
|
||||
|
||||
/// Straight RGBA to a premultiplied pixmap.
|
||||
fn pixmap(img: &image::RgbaImage) -> Option<Arc<Pixmap>> {
|
||||
let (w, h) = img.dimensions();
|
||||
if w > u16::MAX as u32 || h > u16::MAX as u32 {
|
||||
return None;
|
||||
}
|
||||
let data: Vec<vello_cpu::color::PremulRgba8> = img
|
||||
.pixels()
|
||||
.map(|p| {
|
||||
let a = p[3] as u16;
|
||||
let m = |c: u8| ((c as u16 * a + 127) / 255) as u8;
|
||||
vello_cpu::color::PremulRgba8 { r: m(p[0]), g: m(p[1]), b: m(p[2]), a: p[3] }
|
||||
})
|
||||
.collect();
|
||||
Some(Arc::new(Pixmap::from_parts(data, w as u16, h as u16)))
|
||||
}
|
||||
|
||||
/// Decode every frame of an animated GIF (`None` for a still or broken one).
|
||||
fn decode_anim(bytes: &[u8], adj: &PictureAdjust) -> Option<Anim> {
|
||||
use image::{AnimationDecoder, ImageDecoder};
|
||||
let mut dec = image::codecs::gif::GifDecoder::new(std::io::Cursor::new(bytes)).ok()?;
|
||||
dec.set_limits(image::Limits::default()).ok()?;
|
||||
let (w, h) = dec.dimensions();
|
||||
if w == 0 || h == 0 || (w as u64) * (h as u64) > GIF_MAX_CANVAS {
|
||||
return None;
|
||||
}
|
||||
let k = (GIF_SIDE as f64 / w.max(h) as f64).min(1.0);
|
||||
let (nw, nh) = (((w as f64 * k).round() as u32).max(1), ((h as f64 * k).round() as u32).max(1));
|
||||
let size = nw as usize * nh as usize * 4;
|
||||
let (mut frames, mut ends, mut t) = (Vec::new(), Vec::new(), 0.0);
|
||||
for f in dec.into_frames() {
|
||||
let Ok(f) = f else { break };
|
||||
if frames.len() >= GIF_MAX_FRAMES || (frames.len() + 1) * size > GIF_BUDGET {
|
||||
log::info!("animated GIF: keeping its first {} frames", frames.len());
|
||||
break;
|
||||
}
|
||||
let (n, d) = f.delay().numer_denom_ms();
|
||||
let ms = if d == 0 { 0.0 } else { n as f64 / d as f64 };
|
||||
// Like browsers: frames of 10 ms or less show for 100 ms.
|
||||
t += if ms <= 10.0 { 0.1 } else { ms / 1000.0 };
|
||||
let mut img = f.into_buffer();
|
||||
if img.dimensions() != (nw, nh) {
|
||||
img = image::imageops::resize(&img, nw, nh, image::imageops::FilterType::Triangle);
|
||||
}
|
||||
apply_adjust(&mut img, adj);
|
||||
frames.push(pixmap(&img)?);
|
||||
ends.push(t);
|
||||
}
|
||||
(frames.len() > 1).then_some(Anim { frames, ends })
|
||||
}
|
||||
|
||||
/// The decoded animation of GIF bytes, once per bytes and adjustments.
|
||||
fn anim(bytes: &Arc<Vec<u8>>, adj: &PictureAdjust) -> Option<Arc<Anim>> {
|
||||
if !bytes.starts_with(b"GIF8") {
|
||||
return None;
|
||||
}
|
||||
let key = (Arc::as_ptr(bytes) as usize, adjust_key(adj));
|
||||
if let Some(a) = with(|c| c.anims.get(&key).map(|e| e.1.clone())) {
|
||||
return a;
|
||||
}
|
||||
let a = decode_anim(bytes, adj).map(Arc::new);
|
||||
with(|c| {
|
||||
let held: usize = c.anims.values().filter_map(|e| e.1.as_ref()).map(|a| a.bytes()).sum();
|
||||
if held + a.as_ref().map_or(0, |a| a.bytes()) > GIF_CACHE_BUDGET || c.anims.len() >= 64 {
|
||||
c.anims.clear();
|
||||
}
|
||||
c.anims.insert(key, (bytes.clone(), a.clone()));
|
||||
});
|
||||
a
|
||||
}
|
||||
|
||||
/// For an animated GIF: the frame showing `t` seconds into its (looping) play and the seconds until
|
||||
/// the next frame. `None` for anything else.
|
||||
pub fn gif_frame(bytes: &Arc<Vec<u8>>, adj: &PictureAdjust, t: f64) -> Option<(usize, f64)> {
|
||||
anim(bytes, adj).map(|a| a.at(t))
|
||||
}
|
||||
|
||||
/// The frame of an animated GIF at `t` seconds.
|
||||
pub(crate) fn gif_pixmap(bytes: &Arc<Vec<u8>>, adj: &PictureAdjust, t: f64) -> Option<Arc<Pixmap>> {
|
||||
let a = anim(bytes, adj)?;
|
||||
a.frames.get(a.at(t).0).cloned()
|
||||
}
|
||||
|
||||
fn apply_adjust(img: &mut image::RgbaImage, a: &PictureAdjust) {
|
||||
let sat = a.saturation.unwrap_or(1.0);
|
||||
if a.brightness == 0.0 && a.contrast == 0.0 && sat == 1.0 && !a.grayscale && a.clear_color.is_none() && a.duotone.is_none() {
|
||||
@@ -152,3 +259,24 @@ fn apply_adjust(img: &mut image::RgbaImage, a: &PictureAdjust) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#![allow(clippy::expect_used)]
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_gif_of_many_tiny_frames_keeps_only_the_frame_cap() {
|
||||
use image::{Delay, Frame, Rgba, RgbaImage};
|
||||
let frames = (0..GIF_MAX_FRAMES + 200).map(|i| {
|
||||
let c = if i % 2 == 0 { [255, 0, 0, 255] } else { [0, 0, 255, 255] };
|
||||
Frame::from_parts(RgbaImage::from_pixel(1, 1, Rgba(c)), 0, 0, Delay::from_numer_denom_ms(20, 1))
|
||||
});
|
||||
let mut out = Vec::new();
|
||||
image::codecs::gif::GifEncoder::new(&mut out).encode_frames(frames).expect("gif");
|
||||
let a = decode_anim(&out, &PictureAdjust::default()).expect("animated");
|
||||
assert_eq!(a.frames.len(), GIF_MAX_FRAMES);
|
||||
assert_eq!(a.ends.len(), GIF_MAX_FRAMES);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ use kurbo::{Affine, BezPath, PathEl, Point, Rect, Shape as _, Vec2};
|
||||
use vello_common::filter_effects::{EdgeMode, Filter, FilterPrimitive};
|
||||
use vello_cpu::{RenderContext, Resources, peniko};
|
||||
|
||||
pub use images::decode as decode_image;
|
||||
pub use images::{decode as decode_image, gif_frame};
|
||||
pub use morph_path::PathMorph;
|
||||
pub use morph_text::TextMorph;
|
||||
pub use placed::{Placed, PlacedLink, PlacedText, place_slide};
|
||||
@@ -145,6 +145,8 @@ pub struct RenderOpts<'a> {
|
||||
pub skip: &'a [ShapeId],
|
||||
/// Hide the text of this shape (the UI draws it while editing).
|
||||
pub skip_text: Option<ShapeId>,
|
||||
/// Animated GIF pictures and how far into their play they are (seconds); others show frame 1.
|
||||
pub gif_times: &'a [(ShapeId, f64)],
|
||||
/// Worker threads (0 = this thread; required when effects use filters).
|
||||
pub threads: u16,
|
||||
/// Translate the slide within the output (pixels) and output size override.
|
||||
@@ -164,6 +166,7 @@ impl Default for RenderOpts<'_> {
|
||||
state: None,
|
||||
skip: &[],
|
||||
skip_text: None,
|
||||
gif_times: &[],
|
||||
threads: 0,
|
||||
offset: (0.0, 0.0),
|
||||
size: None,
|
||||
@@ -663,7 +666,8 @@ impl Renderer {
|
||||
ctx.set_transform(m);
|
||||
let bounds = Rect::new(0.0, 0.0, w, h);
|
||||
ctx.push_clip_layer(&outline);
|
||||
paint::picture(ctx, f.pres, pf, bounds, m);
|
||||
let t = f.opts.gif_times.iter().find(|g| g.0 == s.id).map(|g| g.1);
|
||||
paint::picture(ctx, f.pres, pf, bounds, m, t);
|
||||
ctx.pop_layer();
|
||||
}
|
||||
ShapeKind::Media(mc) => {
|
||||
@@ -671,7 +675,7 @@ impl Renderer {
|
||||
let bounds = Rect::new(0.0, 0.0, w, h);
|
||||
if let Some(p) = mc.poster {
|
||||
let pf = deckcraft_model::style::PictureFill { media: p, ..Default::default() };
|
||||
paint::picture(ctx, f.pres, &pf, bounds, m);
|
||||
paint::picture(ctx, f.pres, &pf, bounds, m, None);
|
||||
} else if mc.video {
|
||||
ctx.set_paint(peniko::Color::from_rgba8(20, 20, 24, 255));
|
||||
ctx.fill_rect(&bounds);
|
||||
|
||||
@@ -65,7 +65,7 @@ pub fn fill_path(
|
||||
}
|
||||
Fill::Picture(pf) => {
|
||||
ctx.push_clip_layer(path);
|
||||
picture(ctx, pres, pf, bounds, m);
|
||||
picture(ctx, pres, pf, bounds, m, None);
|
||||
ctx.pop_layer();
|
||||
}
|
||||
Fill::Pattern(p) => {
|
||||
@@ -120,15 +120,17 @@ fn p_rect(x: f64, y: f64, w: f64, h: f64) -> BezPath {
|
||||
}
|
||||
fn p_add(_a: &mut BezPath, _b: &mut BezPath) {}
|
||||
|
||||
/// Draw a picture fill into `bounds` (shape-local) honouring crop and stretch/tile modes.
|
||||
pub fn picture(ctx: &mut RenderContext, pres: &Presentation, pf: &PictureFill, bounds: Rect, m: Affine) {
|
||||
/// Draw a picture fill into `bounds` (shape-local) honouring crop and stretch/tile modes; an animated
|
||||
/// GIF shows its frame at `gif_time` seconds.
|
||||
pub fn picture(ctx: &mut RenderContext, pres: &Presentation, pf: &PictureFill, bounds: Rect, m: Affine, gif_time: Option<f64>) {
|
||||
let Some(item) = pres.media(pf.media) else {
|
||||
missing(ctx, bounds);
|
||||
return;
|
||||
};
|
||||
let px_scale = crate::scale_of(m);
|
||||
let want = (bounds.width().max(bounds.height()) * px_scale).max(1.0);
|
||||
let Some(pm) = images::decode_for(&item.data, want, &pf.adjust) else {
|
||||
let frame = gif_time.and_then(|t| images::gif_pixmap(&item.data, &pf.adjust, t));
|
||||
let Some(pm) = frame.or_else(|| images::decode_for(&item.data, want, &pf.adjust)) else {
|
||||
missing(ctx, bounds);
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -273,6 +273,39 @@ fn morph_text_ends_match_the_slides() {
|
||||
assert!(ink > 50, "ink {ink}");
|
||||
}
|
||||
|
||||
/// Two 8×8 frames, red then blue, 100 ms each.
|
||||
fn two_frame_gif() -> Vec<u8> {
|
||||
use image::{Delay, Frame, Rgba, RgbaImage};
|
||||
let frame = |c: [u8; 4]| Frame::from_parts(RgbaImage::from_pixel(8, 8, Rgba(c)), 0, 0, Delay::from_numer_denom_ms(100, 1));
|
||||
let mut out = Vec::new();
|
||||
image::codecs::gif::GifEncoder::new(&mut out).encode_frames([frame([255, 0, 0, 255]), frame([0, 0, 255, 255])]).expect("gif");
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn animated_gif_shows_the_frame_for_its_time() {
|
||||
let mut p = Presentation::default();
|
||||
let media = p.add_media("a.gif", "image/gif", two_frame_gif());
|
||||
let fill = deckcraft_model::style::PictureFill { media, ..Default::default() };
|
||||
let sh = Shape { id: ShapeId(600), xfrm: Some(Xfrm::new(0.0, 0.0, 960.0, 540.0)), kind: ShapeKind::Picture { fill }, ..Default::default() };
|
||||
std::sync::Arc::make_mut(&mut p.slides[0]).shapes = vec![sh];
|
||||
let at = |t: Option<f64>| {
|
||||
let times: Vec<(ShapeId, f64)> = t.map(|t| vec![(ShapeId(600), t)]).unwrap_or_default();
|
||||
render_slide(&p, 0, &RenderOpts { scale: 0.1, gif_times: ×, ..Default::default() }).pixel(48, 27)
|
||||
};
|
||||
let red = |px: [u8; 4]| px[0] > 200 && px[2] < 60;
|
||||
let blue = |px: [u8; 4]| px[2] > 200 && px[0] < 60;
|
||||
assert!(red(at(None)), "no time: the first frame, as exports draw it");
|
||||
assert!(red(at(Some(0.05))));
|
||||
assert!(blue(at(Some(0.15))));
|
||||
assert!(red(at(Some(0.25))), "it loops");
|
||||
let bytes = p.media(media).map(|m| m.data.clone()).expect("media");
|
||||
let adj = Default::default();
|
||||
assert_eq!(gif_frame(&bytes, &adj, 0.15).map(|f| f.0), Some(1));
|
||||
assert_eq!(gif_frame(&bytes, &adj, f64::NAN).map(|f| f.0), Some(0));
|
||||
assert!(gif_frame(&std::sync::Arc::new(b"\x89PNG".to_vec()), &adj, 0.0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn morph_path_ends_match_the_slides() {
|
||||
let from = Xfrm::new(100.0, 100.0, 200.0, 200.0);
|
||||
|
||||
Reference in New Issue
Block a user