From eb95f732e5ae2aabda0443ec941173dc2960d9bc Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Wed, 14 Oct 2026 11:33:49 -0400 Subject: [PATCH 1/3] Media: millisecond-rounded MP4 frame times read at the rate they average to (#1442) Phones and messaging apps store frame times rounded to whole milliseconds (33/34 ms at 30 fps, 16/17 ms at 60 fps). FilmCraft's reader takes the median duration, so such clips imported at 30.303 or 58.824 fps, and New Comp from Selection and renders followed that rate. When the median is no standard rate, the durations near it are averaged and snapped to the standard rate within 0.5 %; exact and variable rates keep the reader's estimate. --- crates/media/src/probe.rs | 54 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index bae714c8..dad34fd1 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -19,8 +19,10 @@ pub fn probe(path: impl AsRef) -> Result { // A movie is read in place (its index), not loaded whole. if let Some(src) = crate::stream::probe_source(path) { let mut f = footage_from_info(&path.to_string_lossy(), src?.info()); - let turn = crate::stream::movie_header(path).map_or(Orientation::NoTransforms, |h| video_turn(&h)); + let header = crate::stream::movie_header(path); + let turn = header.as_deref().map_or(Orientation::NoTransforms, video_turn); turn_footage(&mut f, turn); + correct_rate(&mut f, header.as_deref().and_then(video_rate)); return Ok(f); } let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; @@ -95,12 +97,24 @@ pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { } let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let turn = video_turn(&bytes); + let rate = video_rate(&bytes); let src = filmcraft_codecs::open_bytes(&name, bytes)?; let mut f = footage_from_info(path, src.info()); turn_footage(&mut f, turn); + correct_rate(&mut f, rate); Ok(f) } +/// A movie's frame rate as its frame durations average to ([`video_rate`]), where the reader's +/// own estimate misreads it. +fn correct_rate(f: &mut Footage, rate: Option) { + if f.kind == FootageKind::Video + && let Some(rate) = rate + { + f.frame_rate = rate; + } +} + /// The size as displayed: a movie shot with the camera on its side trades width and height. fn turn_footage(f: &mut Footage, turn: Orientation) { if swaps_sides(turn) { @@ -254,3 +268,41 @@ pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { (false, _, false) => Orientation::Rotate270, } } + +/// The frame rate of an MP4/MOV movie's video track when the reader's own estimate, the median +/// frame duration, misreads it. Phones and messaging apps store frame times rounded to whole +/// milliseconds: a 30 fps clip's frames last 33 or 34 ms and a 60 fps clip's 16 or 17 ms, whose +/// median reads as 30.303 or 58.824 fps. `None` for other containers and for movies the median +/// reads right. +fn video_rate(bytes: &[u8]) -> Option { + let file = filmcraft_isobmff::open(bytes).ok()?; + let t = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video && !t.samples.is_empty())?; + averaged_rate(t.timescale, t.samples.iter().take(240).map(|s| s.duration)) +} + +/// The standard rate ([`FrameRate::COMMON`]) within 0.5 % of what the frame durations near their +/// median average to (`durations` in `1 / timescale` s); `None` when the median duration is a +/// standard rate already, or when the average is no standard rate (variable or unusual rates keep +/// the reader's estimate). +fn averaged_rate(timescale: u32, durations: impl Iterator) -> Option { + let ts = i128::from(timescale); + let mut durs: Vec = durations.filter(|&d| d > 0).map(u64::from).collect(); + durs.sort_unstable(); + let median = *durs.get(durs.len() / 2)?; + if FrameRate::COMMON.iter().any(|r| ts * i128::from(r.den) == i128::from(median) * i128::from(r.num)) { + return None; + } + // Durations within a quarter of the median (widened, so extreme durations can't overflow). + let (n, sum) = durs.iter().filter(|&&d| d.abs_diff(median).saturating_mul(4) <= median).fold((0u64, 0u64), |(n, s), &d| (n + 1, s.saturating_add(d))); + if sum == 0 { + return None; + } + let fps = timescale as f64 * n as f64 / sum as f64; + FrameRate::COMMON + .into_iter() + .map(|r| (r, (fps / r.as_f64() - 1.0).abs())) + .filter(|&(_, off)| off <= 0.005) + .min_by(|a, b| a.1.total_cmp(&b.1)) + .map(|(r, _)| r) +} + From 36a905293b65d4487528456cf6d308e89fd7283d Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Wed, 14 Oct 2026 11:33:49 -0400 Subject: [PATCH 2/3] Media: tests for the averaged MP4 frame rate (#1442) --- crates/media/src/probe.rs | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index dad34fd1..2793be25 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -306,3 +306,48 @@ fn averaged_rate(timescale: u32, durations: impl Iterator) -> Option .map(|(r, _)| r) } +#[cfg(test)] +mod tests { + use super::*; + + /// Durations (in 1/`timescale` s) of `n` frames at `rate` whose times are rounded to the timescale. + fn rounded(rate: FrameRate, timescale: i64, n: i64) -> Vec { + let at = |k: i64| (2 * k * timescale * rate.den + rate.num) / (2 * rate.num); + (0..n).map(|k| (at(k + 1) - at(k)) as u32).collect() + } + + #[test] + fn millisecond_rounded_frame_times_average_to_their_rate() { + assert_eq!(rounded(FrameRate::FPS_30, 1000, 6), [33, 34, 33, 33, 34, 33]); + assert_eq!(averaged_rate(1000, rounded(FrameRate::FPS_30, 1000, 120).into_iter()), Some(FrameRate::FPS_30)); + assert_eq!(averaged_rate(1000, rounded(FrameRate::FPS_60, 1000, 240).into_iter()), Some(FrameRate::FPS_60)); + // NTSC rates stay NTSC, not the round rate next to them. + for r in [FrameRate::FPS_23_976, FrameRate::FPS_29_97, FrameRate::FPS_59_94] { + assert_eq!(averaged_rate(1000, rounded(r, 1000, 240).into_iter()), Some(r), "{r:?}"); + } + // A short clip: its few frames average to the rate as well. + assert_eq!(averaged_rate(1000, [33, 34, 33].into_iter()), Some(FrameRate::FPS_30)); + } + + #[test] + fn exact_and_variable_rates_keep_the_readers_estimate() { + // A median that is a standard rate already reads right. + assert_eq!(averaged_rate(30000, rounded(FrameRate::FPS_29_97, 30000, 240).into_iter()), None); + assert_eq!(averaged_rate(90000, [3000; 10].into_iter()), None); + assert_eq!(averaged_rate(600, [20, 20, 19, 21, 20].into_iter()), None); + // Durations averaging to no standard rate (22.2 fps). + assert_eq!(averaged_rate(1000, [40, 45, 50].repeat(20).into_iter()), None); + // A truly 30.3 fps clip is not pulled to 30 (1 % off). + assert_eq!(averaged_rate(1000, [33; 30].into_iter()), None); + } + + #[test] + fn degenerate_durations_have_no_rate() { + assert_eq!(averaged_rate(1000, std::iter::empty()), None); + assert_eq!(averaged_rate(1000, [0, 0, 0].into_iter()), None); + assert_eq!(averaged_rate(0, [33, 34, 33].into_iter()), None); + assert_eq!(averaged_rate(u32::MAX, [u32::MAX, u32::MAX - 1, 1, 0].into_iter()), None); + assert_eq!(averaged_rate(u32::MAX, [u32::MAX; 300].into_iter()), None); + assert_eq!(video_rate(b"not a movie"), None); + } +} From 8b536e597bab07116fb2776f23473549e868e329 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Wed, 14 Oct 2026 11:37:36 -0400 Subject: [PATCH 3/3] Export: millisecond-rounded footage renders at its rate, each frame once (#1442) --- crates/export/tests/ms_rounded_footage.rs | 148 ++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 crates/export/tests/ms_rounded_footage.rs diff --git a/crates/export/tests/ms_rounded_footage.rs b/crates/export/tests/ms_rounded_footage.rs new file mode 100644 index 00000000..3d688023 --- /dev/null +++ b/crates/export/tests/ms_rounded_footage.rs @@ -0,0 +1,148 @@ +//! Footage whose frame times are rounded to whole milliseconds (#1442), as phones and messaging +//! apps write MP4s: a 30 fps clip's frames last 33 or 34 ms, a 60 fps clip's 16 or 17 ms. It +//! imports at its true rate, a comp made from it (New Comp from Selection takes the footage's +//! rate) renders every source frame once, and the H.264 output has that rate and frame count. +//! +//! The fixtures are written here with FilmCraft's own encoder and muxer: 128×32 frames showing +//! their index in binary (eight bars, white for a set bit). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::{OutputFormat, OutputModule, RenderSettings, TimeSpan}; +use effectcraft_export::{Job, export}; +use effectcraft_media::MediaPool; +use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project, build}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use filmcraft_isobmff::{AvcConfig, Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + +const W: u32 = 128; +const H: u32 = 32; +const BARS: u32 = 8; + +fn out_dir(name: &str) -> PathBuf { + let d = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/export").join(name); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("mkdir"); + d +} + +/// Durations of `n` frames at `rate` with their times rounded to whole milliseconds. +fn ms_rounded(rate: FrameRate, n: i64) -> Vec { + let at = |k: i64| (2 * k * 1000 * rate.den + rate.num) / (2 * rate.num); + (0..n).map(|k| u32::try_from(at(k + 1) - at(k)).expect("duration")).collect() +} + +/// An H.264 MP4 (video track timescale 1000) with one frame per entry of `durations` (ms); +/// frame k shows k in binary. +fn write_movie(path: &Path, durations: &[u32]) { + let mut cfg = filmcraft_h264enc::EncoderConfig::new(W, H, 30, 1); + cfg.format = filmcraft_h264enc::PacketFormat::LengthPrefixed; + cfg.aud = false; + cfg.bframes = 0; + cfg.keyint = 30; + cfg.scenecut = false; + let mut enc = filmcraft_h264enc::Encoder::new(cfg).expect("encoder"); + let (u, v) = (vec![128u8; (W * H / 4) as usize], vec![128u8; (W * H / 4) as usize]); + let mut packets = Vec::new(); + for k in 0..durations.len() { + let y: Vec = (0..W * H).map(|i| if (k >> (i % W * BARS / W)) & 1 == 1 { 235 } else { 16 }).collect(); + let frame = filmcraft_h264enc::YuvFrame { y: &y, u: &u, v: &v, y_stride: W as usize, uv_stride: W as usize / 2 }; + packets.extend(enc.try_encode(&frame, k as i64).expect("encode")); + } + packets.extend(enc.flush()); + assert_eq!(packets.len(), durations.len(), "one access unit per frame"); + let avcc = AvcConfig::parse(&enc.avcc()).expect("avcC"); + let file = std::fs::File::create(path).expect("create"); + let mut mux = Mp4Writer::new(file, WriterOptions::new(Brand::Mp4)).expect("mux"); + let t = mux.add_track(TrackConfig::new(SampleEntry::avc(avcc, W as u16, H as u16), 1000)).expect("track"); + for (p, &duration) in packets.iter().zip(durations) { + mux.write_sample(t, WriteSample { data: &p.data, duration, composition_offset: 0, is_sync: p.keyframe }).expect("sample"); + } + mux.finish().expect("finish"); +} + +/// The frame index a decoded frame shows. +fn shown(img: &Image) -> u32 { + (0..BARS).filter(|b| img.data[img.idx(b * W / BARS + W / BARS / 2, H / 2)][1] > 0.5).map(|b| 1 << b).sum() +} + +fn frame_count(f: &Footage) -> i64 { + f.frame_rate.frame_at(f.duration - Tick(1)) + 1 +} + +/// Every frame of a movie in order, as indices. +fn frames_of(f: &Footage) -> Vec { + let pool = MediaPool::new(); + (0..frame_count(f)).map(|k| shown(&pool.frame_at(f, f.frame_rate.tick_of(k)).expect("decode"))).collect() +} + +/// Import `path`, make a comp from it as New Comp from Selection does (the footage's size, rate +/// and duration), render it to H.264 and return the output's footage. +fn comp_from_footage(path: &Path, out: &Path) -> Footage { + let footage = effectcraft_media::probe(path).expect("probe"); + let mut p = Project::default(); + let fid = p.add_item("clip.mp4", Label::Aqua, None, ItemKind::Footage(footage.clone())); + let rate = p.item(fid).and_then(|it| it.frame_rate()).expect("a movie has a rate"); + let mut comp = Comp::new(W, H, rate, footage.duration); + comp.layers.push(build::layer(&mut p, &comp, "clip.mp4", LayerSource::Footage { item: fid }, (W, H), None)); + let cid = p.add_item("clip", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let pool = MediaPool::new(); + let settings = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let om = OutputModule { bitrate_kbps: 4000, ..OutputModule::for_format(OutputFormat::H264) }; + let out_path = out.to_string_lossy().to_string(); + let job = Job { + project: &p, + footage: &pool, + expr: None, + accel: None, + comp: cid, + settings: &settings, + output: &om, + path: &out_path, + sink: None, + options: Default::default(), + nested_switches: true, + }; + let r = export(&job, &mut |_| true).expect("export"); + assert_eq!(r.frames as i64, frame_count(&footage), "the render has the footage's frames"); + effectcraft_media::probe(out).expect("probe the render") +} + +#[test] +fn millisecond_rounded_footage_keeps_its_rate_through_comp_and_render() { + let d = out_dir("ms_rounded"); + for (rate, frames) in [(FrameRate::FPS_30, 120), (FrameRate::FPS_60, 240)] { + let path = d.join(format!("ms-{}.mp4", rate.num)); + write_movie(&path, &ms_rounded(rate, frames)); + let f = effectcraft_media::probe(&path).expect("probe"); + assert_eq!((f.frame_rate, frame_count(&f)), (rate, frames), "{rate:?} footage"); + let bytes: Arc<[u8]> = std::fs::read(&path).expect("read").into(); + let from_bytes = effectcraft_media::probe_bytes(&path.to_string_lossy(), bytes).expect("probe bytes"); + assert_eq!(from_bytes.frame_rate, rate, "{rate:?} footage read from memory"); + let all: Vec = (0..frames as u32).collect(); + assert_eq!(frames_of(&f), all, "{rate:?} footage: each source frame once"); + + let out = comp_from_footage(&path, &d.join(format!("render-{}.mp4", rate.num))); + assert_eq!((out.frame_rate, frame_count(&out)), (rate, frames), "{rate:?} render"); + assert_eq!(out.duration, Tick::from_seconds_f64(4.0), "{rate:?} render duration"); + assert_eq!(frames_of(&out), all, "{rate:?} render: no frame skipped or repeated"); + } +} + +#[test] +fn ntsc_and_variable_rate_footage_is_not_pulled_to_a_round_rate() { + let d = out_dir("ms_rounded_controls"); + for rate in [FrameRate::FPS_23_976, FrameRate::FPS_29_97, FrameRate::FPS_59_94] { + let path = d.join(format!("ms-{}.mp4", rate.num)); + write_movie(&path, &ms_rounded(rate, 60)); + assert_eq!(effectcraft_media::probe(&path).expect("probe").frame_rate, rate); + } + // Frames of 40, 45 and 50 ms: the reader's median (45 ms, 22.2 fps) stands. + let path = d.join("variable.mp4"); + write_movie(&path, &[40, 45, 50].repeat(20)); + let f = effectcraft_media::probe(&path).expect("probe"); + assert!(!FrameRate::COMMON.contains(&f.frame_rate) && (f.frame_rate.as_f64() - 1000.0 / 45.0).abs() < 0.01, "{:?}", f.frame_rate); +}