diff --git a/crates/export/tests/ms_rounded_footage.rs b/crates/export/tests/ms_rounded_footage.rs new file mode 100644 index 00000000..3d688023 --- /dev/null +++ b/crates/export/tests/ms_rounded_footage.rs @@ -0,0 +1,148 @@ +//! Footage whose frame times are rounded to whole milliseconds (#1442), as phones and messaging +//! apps write MP4s: a 30 fps clip's frames last 33 or 34 ms, a 60 fps clip's 16 or 17 ms. It +//! imports at its true rate, a comp made from it (New Comp from Selection takes the footage's +//! rate) renders every source frame once, and the H.264 output has that rate and frame count. +//! +//! The fixtures are written here with FilmCraft's own encoder and muxer: 128×32 frames showing +//! their index in binary (eight bars, white for a set bit). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::{OutputFormat, OutputModule, RenderSettings, TimeSpan}; +use effectcraft_export::{Job, export}; +use effectcraft_media::MediaPool; +use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project, build}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use filmcraft_isobmff::{AvcConfig, Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + +const W: u32 = 128; +const H: u32 = 32; +const BARS: u32 = 8; + +fn out_dir(name: &str) -> PathBuf { + let d = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/export").join(name); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("mkdir"); + d +} + +/// Durations of `n` frames at `rate` with their times rounded to whole milliseconds. +fn ms_rounded(rate: FrameRate, n: i64) -> Vec { + let at = |k: i64| (2 * k * 1000 * rate.den + rate.num) / (2 * rate.num); + (0..n).map(|k| u32::try_from(at(k + 1) - at(k)).expect("duration")).collect() +} + +/// An H.264 MP4 (video track timescale 1000) with one frame per entry of `durations` (ms); +/// frame k shows k in binary. +fn write_movie(path: &Path, durations: &[u32]) { + let mut cfg = filmcraft_h264enc::EncoderConfig::new(W, H, 30, 1); + cfg.format = filmcraft_h264enc::PacketFormat::LengthPrefixed; + cfg.aud = false; + cfg.bframes = 0; + cfg.keyint = 30; + cfg.scenecut = false; + let mut enc = filmcraft_h264enc::Encoder::new(cfg).expect("encoder"); + let (u, v) = (vec![128u8; (W * H / 4) as usize], vec![128u8; (W * H / 4) as usize]); + let mut packets = Vec::new(); + for k in 0..durations.len() { + let y: Vec = (0..W * H).map(|i| if (k >> (i % W * BARS / W)) & 1 == 1 { 235 } else { 16 }).collect(); + let frame = filmcraft_h264enc::YuvFrame { y: &y, u: &u, v: &v, y_stride: W as usize, uv_stride: W as usize / 2 }; + packets.extend(enc.try_encode(&frame, k as i64).expect("encode")); + } + packets.extend(enc.flush()); + assert_eq!(packets.len(), durations.len(), "one access unit per frame"); + let avcc = AvcConfig::parse(&enc.avcc()).expect("avcC"); + let file = std::fs::File::create(path).expect("create"); + let mut mux = Mp4Writer::new(file, WriterOptions::new(Brand::Mp4)).expect("mux"); + let t = mux.add_track(TrackConfig::new(SampleEntry::avc(avcc, W as u16, H as u16), 1000)).expect("track"); + for (p, &duration) in packets.iter().zip(durations) { + mux.write_sample(t, WriteSample { data: &p.data, duration, composition_offset: 0, is_sync: p.keyframe }).expect("sample"); + } + mux.finish().expect("finish"); +} + +/// The frame index a decoded frame shows. +fn shown(img: &Image) -> u32 { + (0..BARS).filter(|b| img.data[img.idx(b * W / BARS + W / BARS / 2, H / 2)][1] > 0.5).map(|b| 1 << b).sum() +} + +fn frame_count(f: &Footage) -> i64 { + f.frame_rate.frame_at(f.duration - Tick(1)) + 1 +} + +/// Every frame of a movie in order, as indices. +fn frames_of(f: &Footage) -> Vec { + let pool = MediaPool::new(); + (0..frame_count(f)).map(|k| shown(&pool.frame_at(f, f.frame_rate.tick_of(k)).expect("decode"))).collect() +} + +/// Import `path`, make a comp from it as New Comp from Selection does (the footage's size, rate +/// and duration), render it to H.264 and return the output's footage. +fn comp_from_footage(path: &Path, out: &Path) -> Footage { + let footage = effectcraft_media::probe(path).expect("probe"); + let mut p = Project::default(); + let fid = p.add_item("clip.mp4", Label::Aqua, None, ItemKind::Footage(footage.clone())); + let rate = p.item(fid).and_then(|it| it.frame_rate()).expect("a movie has a rate"); + let mut comp = Comp::new(W, H, rate, footage.duration); + comp.layers.push(build::layer(&mut p, &comp, "clip.mp4", LayerSource::Footage { item: fid }, (W, H), None)); + let cid = p.add_item("clip", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let pool = MediaPool::new(); + let settings = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let om = OutputModule { bitrate_kbps: 4000, ..OutputModule::for_format(OutputFormat::H264) }; + let out_path = out.to_string_lossy().to_string(); + let job = Job { + project: &p, + footage: &pool, + expr: None, + accel: None, + comp: cid, + settings: &settings, + output: &om, + path: &out_path, + sink: None, + options: Default::default(), + nested_switches: true, + }; + let r = export(&job, &mut |_| true).expect("export"); + assert_eq!(r.frames as i64, frame_count(&footage), "the render has the footage's frames"); + effectcraft_media::probe(out).expect("probe the render") +} + +#[test] +fn millisecond_rounded_footage_keeps_its_rate_through_comp_and_render() { + let d = out_dir("ms_rounded"); + for (rate, frames) in [(FrameRate::FPS_30, 120), (FrameRate::FPS_60, 240)] { + let path = d.join(format!("ms-{}.mp4", rate.num)); + write_movie(&path, &ms_rounded(rate, frames)); + let f = effectcraft_media::probe(&path).expect("probe"); + assert_eq!((f.frame_rate, frame_count(&f)), (rate, frames), "{rate:?} footage"); + let bytes: Arc<[u8]> = std::fs::read(&path).expect("read").into(); + let from_bytes = effectcraft_media::probe_bytes(&path.to_string_lossy(), bytes).expect("probe bytes"); + assert_eq!(from_bytes.frame_rate, rate, "{rate:?} footage read from memory"); + let all: Vec = (0..frames as u32).collect(); + assert_eq!(frames_of(&f), all, "{rate:?} footage: each source frame once"); + + let out = comp_from_footage(&path, &d.join(format!("render-{}.mp4", rate.num))); + assert_eq!((out.frame_rate, frame_count(&out)), (rate, frames), "{rate:?} render"); + assert_eq!(out.duration, Tick::from_seconds_f64(4.0), "{rate:?} render duration"); + assert_eq!(frames_of(&out), all, "{rate:?} render: no frame skipped or repeated"); + } +} + +#[test] +fn ntsc_and_variable_rate_footage_is_not_pulled_to_a_round_rate() { + let d = out_dir("ms_rounded_controls"); + for rate in [FrameRate::FPS_23_976, FrameRate::FPS_29_97, FrameRate::FPS_59_94] { + let path = d.join(format!("ms-{}.mp4", rate.num)); + write_movie(&path, &ms_rounded(rate, 60)); + assert_eq!(effectcraft_media::probe(&path).expect("probe").frame_rate, rate); + } + // Frames of 40, 45 and 50 ms: the reader's median (45 ms, 22.2 fps) stands. + let path = d.join("variable.mp4"); + write_movie(&path, &[40, 45, 50].repeat(20)); + let f = effectcraft_media::probe(&path).expect("probe"); + assert!(!FrameRate::COMMON.contains(&f.frame_rate) && (f.frame_rate.as_f64() - 1000.0 / 45.0).abs() < 0.01, "{:?}", f.frame_rate); +} diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index bae714c8..2793be25 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -19,8 +19,10 @@ pub fn probe(path: impl AsRef) -> Result { // A movie is read in place (its index), not loaded whole. if let Some(src) = crate::stream::probe_source(path) { let mut f = footage_from_info(&path.to_string_lossy(), src?.info()); - let turn = crate::stream::movie_header(path).map_or(Orientation::NoTransforms, |h| video_turn(&h)); + let header = crate::stream::movie_header(path); + let turn = header.as_deref().map_or(Orientation::NoTransforms, video_turn); turn_footage(&mut f, turn); + correct_rate(&mut f, header.as_deref().and_then(video_rate)); return Ok(f); } let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; @@ -95,12 +97,24 @@ pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { } let name = Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let turn = video_turn(&bytes); + let rate = video_rate(&bytes); let src = filmcraft_codecs::open_bytes(&name, bytes)?; let mut f = footage_from_info(path, src.info()); turn_footage(&mut f, turn); + correct_rate(&mut f, rate); Ok(f) } +/// A movie's frame rate as its frame durations average to ([`video_rate`]), where the reader's +/// own estimate misreads it. +fn correct_rate(f: &mut Footage, rate: Option) { + if f.kind == FootageKind::Video + && let Some(rate) = rate + { + f.frame_rate = rate; + } +} + /// The size as displayed: a movie shot with the camera on its side trades width and height. fn turn_footage(f: &mut Footage, turn: Orientation) { if swaps_sides(turn) { @@ -254,3 +268,86 @@ pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { (false, _, false) => Orientation::Rotate270, } } + +/// The frame rate of an MP4/MOV movie's video track when the reader's own estimate, the median +/// frame duration, misreads it. Phones and messaging apps store frame times rounded to whole +/// milliseconds: a 30 fps clip's frames last 33 or 34 ms and a 60 fps clip's 16 or 17 ms, whose +/// median reads as 30.303 or 58.824 fps. `None` for other containers and for movies the median +/// reads right. +fn video_rate(bytes: &[u8]) -> Option { + let file = filmcraft_isobmff::open(bytes).ok()?; + let t = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video && !t.samples.is_empty())?; + averaged_rate(t.timescale, t.samples.iter().take(240).map(|s| s.duration)) +} + +/// The standard rate ([`FrameRate::COMMON`]) within 0.5 % of what the frame durations near their +/// median average to (`durations` in `1 / timescale` s); `None` when the median duration is a +/// standard rate already, or when the average is no standard rate (variable or unusual rates keep +/// the reader's estimate). +fn averaged_rate(timescale: u32, durations: impl Iterator) -> Option { + let ts = i128::from(timescale); + let mut durs: Vec = durations.filter(|&d| d > 0).map(u64::from).collect(); + durs.sort_unstable(); + let median = *durs.get(durs.len() / 2)?; + if FrameRate::COMMON.iter().any(|r| ts * i128::from(r.den) == i128::from(median) * i128::from(r.num)) { + return None; + } + // Durations within a quarter of the median (widened, so extreme durations can't overflow). + let (n, sum) = durs.iter().filter(|&&d| d.abs_diff(median).saturating_mul(4) <= median).fold((0u64, 0u64), |(n, s), &d| (n + 1, s.saturating_add(d))); + if sum == 0 { + return None; + } + let fps = timescale as f64 * n as f64 / sum as f64; + FrameRate::COMMON + .into_iter() + .map(|r| (r, (fps / r.as_f64() - 1.0).abs())) + .filter(|&(_, off)| off <= 0.005) + .min_by(|a, b| a.1.total_cmp(&b.1)) + .map(|(r, _)| r) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Durations (in 1/`timescale` s) of `n` frames at `rate` whose times are rounded to the timescale. + fn rounded(rate: FrameRate, timescale: i64, n: i64) -> Vec { + let at = |k: i64| (2 * k * timescale * rate.den + rate.num) / (2 * rate.num); + (0..n).map(|k| (at(k + 1) - at(k)) as u32).collect() + } + + #[test] + fn millisecond_rounded_frame_times_average_to_their_rate() { + assert_eq!(rounded(FrameRate::FPS_30, 1000, 6), [33, 34, 33, 33, 34, 33]); + assert_eq!(averaged_rate(1000, rounded(FrameRate::FPS_30, 1000, 120).into_iter()), Some(FrameRate::FPS_30)); + assert_eq!(averaged_rate(1000, rounded(FrameRate::FPS_60, 1000, 240).into_iter()), Some(FrameRate::FPS_60)); + // NTSC rates stay NTSC, not the round rate next to them. + for r in [FrameRate::FPS_23_976, FrameRate::FPS_29_97, FrameRate::FPS_59_94] { + assert_eq!(averaged_rate(1000, rounded(r, 1000, 240).into_iter()), Some(r), "{r:?}"); + } + // A short clip: its few frames average to the rate as well. + assert_eq!(averaged_rate(1000, [33, 34, 33].into_iter()), Some(FrameRate::FPS_30)); + } + + #[test] + fn exact_and_variable_rates_keep_the_readers_estimate() { + // A median that is a standard rate already reads right. + assert_eq!(averaged_rate(30000, rounded(FrameRate::FPS_29_97, 30000, 240).into_iter()), None); + assert_eq!(averaged_rate(90000, [3000; 10].into_iter()), None); + assert_eq!(averaged_rate(600, [20, 20, 19, 21, 20].into_iter()), None); + // Durations averaging to no standard rate (22.2 fps). + assert_eq!(averaged_rate(1000, [40, 45, 50].repeat(20).into_iter()), None); + // A truly 30.3 fps clip is not pulled to 30 (1 % off). + assert_eq!(averaged_rate(1000, [33; 30].into_iter()), None); + } + + #[test] + fn degenerate_durations_have_no_rate() { + assert_eq!(averaged_rate(1000, std::iter::empty()), None); + assert_eq!(averaged_rate(1000, [0, 0, 0].into_iter()), None); + assert_eq!(averaged_rate(0, [33, 34, 33].into_iter()), None); + assert_eq!(averaged_rate(u32::MAX, [u32::MAX, u32::MAX - 1, 1, 0].into_iter()), None); + assert_eq!(averaged_rate(u32::MAX, [u32::MAX; 300].into_iter()), None); + assert_eq!(video_rate(b"not a movie"), None); + } +}