filmcraft/crates/codecs/tests/it/mp4_oracle.rs

//! MP4 audio against ffmpeg: ffmpeg writes an AAC track (with the edit list that skips the encoder
//! priming), and reads at another sample rate must land on the same source samples as the native
//! decode, however the requests are cut.

use crate::common;

use common::*;

/// A source whose rate differs from the sequence's is resampled per request. An AAC track in MP4
/// carries an edit list that skips the encoder priming (ffmpeg: 1024 samples), and every request
/// must still land on the same source samples: reading at 44.1 kHz in video-frame-sized pieces (as
/// an export does) equals the 48 kHz decode interpolated at the same times.
#[test]
fn aac_mp4_resampled_reads_follow_the_edit_list() {
    let ff = filmcraft_testkit::require_ffmpeg!();
    let Some(f) = fixture(&ff, "tone_aac_48k.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=3", "-c:a", "aac", "-b:a", "128k"]) else {
        panic!("could not write the AAC fixture")
    };
    let src = filmcraft_codecs::open_bytes("tone_aac_48k.mp4", bytes(&f)).unwrap();
    let native = src.audio(0, 2 * 48_000, 48_000).unwrap().channels[0].clone();
    let ratio = 48_000.0 / 44_100.0;
    // 30 fps at 44.1 kHz, starting mid-file and off the 48 kHz grid
    let (mut start, piece) = (22_051i64, 1470usize);
    let (mut worst, mut peak) = (0f32, 0f32);
    for _ in 0..20 {
        let got = src.audio(start, piece, 44_100).unwrap();
        for (k, &v) in got.channels[0].iter().enumerate() {
            let pos = (start + k as i64) as f64 * ratio;
            let i0 = pos.floor() as usize;
            let want = native[i0] + (native[i0 + 1] - native[i0]) * (pos - i0 as f64) as f32;
            worst = worst.max((v - want).abs());
            peak = peak.max(v.abs());
        }
        start += piece as i64;
    }
    let level = native.iter().fold(0f32, |m, v| m.max(v.abs()));
    assert!(peak <= level * 1.01, "resampled peak {peak} above the source's {level}");
    assert!(worst < 1e-4, "resampled read differs from the source by {worst}");
}

/// ffmpeg's MP4 muxer skips the B-frame reorder delay with an edit list, so the media (`mdhd`) runs
/// one frame past what plays. The clip must be as long as ffmpeg decodes it: 60 frames, not 61.
#[test]
fn bframe_mp4_duration_is_the_decoded_frame_count() {
    let ff = filmcraft_testkit::require_ffmpeg!();
    let args = ["-f", "lavfi", "-i", "testsrc2=size=160x96:rate=30:duration=2", "-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p"];
    let Some(f) = fixture(&ff, "bframes_x264.mp4", &args) else {
        eprintln!("SKIPPED: ffmpeg without libx264");
        return;
    };
    let n = ffmpeg_frames(&ff, &f, "yuv420p").len() / (160 * 96 * 3 / 2);
    assert_eq!(n, 60);
    let src = filmcraft_codecs::open_bytes("bframes_x264.mp4", bytes(&f)).unwrap();
    assert_eq!(src.info().duration, filmcraft_time::FrameRate::FPS_30.tick_of(n as i64));
}

/// `mp4` (moov after mdat, as ffmpeg writes it) with a clean aperture (`clap`) of `w`×`h` moved by
/// (`dx`, `dy`) from the centre, inserted at the end of the first video sample entry.
fn with_clap(mp4: &[u8], (w, h, dx, dy): (u32, u32, i32, i32)) -> Vec<u8> {
    let be = |b: &[u8], at: usize| u32::from_be_bytes([b[at], b[at + 1], b[at + 2], b[at + 3]]) as usize;
    // the chain moov > trak > mdia > minf > stbl > stsd > avc1 (stsd has 8 bytes before its entries)
    let mut chain = Vec::new();
    let (mut off, mut end) = (0, mp4.len());
    for (want, skip) in [(b"moov", 8), (b"trak", 8), (b"mdia", 8), (b"minf", 8), (b"stbl", 8), (b"stsd", 16), (b"avc1", 0)] {
        loop {
            assert!(off + 8 <= end, "no {}", String::from_utf8_lossy(want));
            let size = be(mp4, off);
            if &mp4[off + 4..off + 8] == want {
                chain.push(off);
                end = off + size;
                off += skip;
                break;
            }
            off += size;
        }
    }
    let entry = *chain.last().unwrap();
    assert!(mp4.windows(4).position(|x| x == b"mdat").unwrap() < chain[0], "moov must follow mdat");
    let mut clap = Vec::new();
    for v in [40u32, u32::from_be_bytes(*b"clap"), w, 1, h, 1, dx as u32, 1, dy as u32, 1] {
        clap.extend_from_slice(&v.to_be_bytes());
    }
    let mut out = mp4.to_vec();
    let at = entry + be(mp4, entry);
    out.splice(at..at, clap);
    for &b in &chain {
        let size = (be(&out, b) + 40) as u32;
        out[b..b + 4].copy_from_slice(&size.to_be_bytes());
    }
    out
}

/// Cropped pictures are the size and the samples ffmpeg decodes (#288): H.264 frame cropping (SPS)
/// and an HEVC conformance window while the sample entry gives the uncropped size, and a clean
/// aperture (`clap`, as iPhones write), centred, off-centre, and with a 90° display rotation
/// applied after it.
#[test]
fn cropped_pictures_match_ffmpeg() {
    let ff = filmcraft_testkit::require_ffmpeg!();
    let src = ["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=30:duration=1,noise=alls=10:allf=t", "-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p"];
    let Some(plain) = fixture(&ff, "crop_plain_x264.mp4", &src) else {
        eprintln!("SKIPPED: ffmpeg without libx264");
        return;
    };
    let sps =
        fixture(&ff, "crop_sps_x264.mp4", &[&src[..], &["-bsf:v", "h264_metadata=crop_left=16:crop_right=16:crop_top=8:crop_bottom=8"]].concat()).unwrap();
    // the display matrix added without re-encoding (an encode would turn the pixels instead)
    let rotated = fixture(&ff, "crop_rotated_x264.mp4", &["-display_rotation", "90", "-i", plain.to_str().unwrap(), "-c", "copy"]).unwrap();
    let dir = dir();
    let mut cases = vec![("SPS crop", sps, (288, 224))];
    // HEVC: the conformance window
    let hevc = [
        "-f",
        "lavfi",
        "-i",
        "testsrc2=size=320x240:rate=30:duration=1,noise=alls=10:allf=t",
        "-c:v",
        "libx265",
        "-x265-params",
        "log-level=error:bframes=3",
        "-tag:v",
        "hvc1",
        "-pix_fmt",
        "yuv420p",
    ];
    match fixture(&ff, "crop_conformance_x265.mp4", &[&hevc[..], &["-bsf:v", "hevc_metadata=crop_left=16:crop_right=16:crop_top=8:crop_bottom=8"]].concat()) {
        Some(p) => cases.push(("HEVC conformance window", p, (288, 224))),
        None => eprintln!("SKIPPED (HEVC case): ffmpeg without libx265"),
    }
    for (name, base, clap, size) in [
        ("centred clap", &plain, (280, 200, 0, 0), (280, 200)),
        ("off-centre clap", &plain, (200, 150, -30, 22), (200, 150)),
        ("clap then rotation", &rotated, (280, 200, 0, 0), (200, 280)),
    ] {
        let path = dir.join(format!("crop_{}.mp4", name.replace(' ', "_")));
        std::fs::write(&path, with_clap(&bytes(base), clap)).unwrap();
        cases.push((name, path, size));
    }
    for (name, path, (w, h)) in cases {
        let src = filmcraft_codecs::open_bytes(path.file_name().unwrap().to_str().unwrap(), bytes(&path)).unwrap();
        let v = src.info().video.clone().unwrap();
        assert_eq!((v.width, v.height), (w, h), "{name}: reported size");
        let (cw, ch) = (w.div_ceil(2) as usize, h.div_ceil(2) as usize);
        // `-flags unaligned`: ffmpeg otherwise skips a left crop that would misalign its frame
        // pointers (on AVX-512 machines it needs 64-byte alignment) and returns wider frames
        let raw = ffmpeg_out(
            &ff,
            &["-flags", "unaligned", "-i", path.to_str().unwrap(), "-map", "0:v:0", "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-"],
        );
        let frame_bytes = (w * h) as usize + 2 * cw * ch;
        if name.contains("clap") && !raw.len().is_multiple_of(frame_bytes) {
            // ffmpeg before 7.1 ignores `clap` and decodes the whole picture
            eprintln!("SKIPPED ({name}): this ffmpeg does not apply the clean aperture");
            continue;
        }
        assert_eq!(raw.len() % frame_bytes, 0, "{name}: ffmpeg's frames are {w}x{h}");
        let n = raw.len() / frame_bytes;
        assert_eq!(n, 30, "{name}");
        for (i, want) in raw.chunks(frame_bytes).enumerate() {
            let f = src.video_frame(filmcraft_media::FrameRequest::full(v.frame_rate.tick_of(i as i64))).unwrap();
            assert_eq!((f.width, f.height), (w, h), "{name}: frame {i} size");
            assert!(planes(&f) == raw_planes(want, w as usize, h as usize, cw, ch, 1), "{name}: frame {i} differs from ffmpeg's");
        }
    }
}

/// An open GOP (x265 `open-gop`, like iPhone HEVC): each CRA after the first is followed in the
/// file by RASL pictures shown before it, predicted from the GOP before. Asked for first (a clip
/// trimmed to start on one, the playhead parked on one), such a picture came out black: decoding
/// started at the CRA after it, which leaves its RASL pictures out. Every frame, asked of a fresh
/// source and in an order that visits each CRA before its leading pictures, is ffmpeg's, from MP4
/// and from Matroska.
#[test]
fn open_gop_leading_pictures_decode_alone() {
    let ff = filmcraft_testkit::require_ffmpeg!();
    let args = [
        "-f",
        "lavfi",
        "-i",
        "testsrc2=size=160x96:rate=30:duration=2,noise=alls=10:allf=t",
        "-c:v",
        "libx265",
        "-x265-params",
        "log-level=error:keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3",
        "-tag:v",
        "hvc1",
        "-pix_fmt",
        "yuv420p",
    ];
    let Some(mp4) = fixture(&ff, "open_gop_x265.mp4", &args) else {
        eprintln!("SKIPPED: ffmpeg without libx265");
        return;
    };
    let mkv = fixture(&ff, "open_gop_x265.mkv", &["-i", mp4.to_str().unwrap(), "-c", "copy"]).unwrap();
    let raw = ffmpeg_frames(&ff, &mp4, "yuv420p");
    let frame_bytes = 160 * 96 * 3 / 2;
    let want: Vec<_> = raw.chunks(frame_bytes).map(|f| raw_planes(f, 160, 96, 80, 48, 1)).collect();
    assert_eq!(want.len(), 60);
    for path in [&mp4, &mkv] {
        let name = path.file_name().unwrap().to_str().unwrap();
        let open = || filmcraft_codecs::open_bytes(name, bytes(path)).unwrap();
        let rate = open().info().frame_rate();
        let frame = |src: &filmcraft_media::SharedSource, i: usize| {
            let f = src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(i as i64))).unwrap_or_else(|e| panic!("{name}: frame {i}: {e}"));
            assert!(planes(&f) == want[i], "{name}: frame {i} differs from ffmpeg's");
        };
        for i in 0..want.len() {
            frame(&open(), i);
        }
        let src = open();
        for cra in [12, 24, 36, 48] {
            for i in [cra, cra - 1, cra - 2, cra - 3] {
                frame(&src, i);
            }
        }
    }
}

/// #800: MP2 in MP4 (`mp4a` + object type 0x6B, which names no layer) and in MOV (QuickTime
/// `.mp2`) decodes sample-exactly as ffmpeg does: same length, the encoder delay (481 samples)
/// trimmed by the edit list, and the same samples from the first one on, also after a seek. It
/// used to go to the layer III decoder (MP4: silence) or not decode at all (MOV).
#[test]
fn mp2_in_mp4_and_mov_matches_ffmpeg() {
    let ff = filmcraft_testkit::require_ffmpeg!();
    for (name, rate) in [("mp2_stereo_224k_48k.mp4", 48_000u32), ("mp2_stereo_224k_44k.mov", 44_100)] {
        let input = format!("anoisesrc=d=1:c=pink:r={rate}:a=0.3");
        let Some(f) = fixture(&ff, name, &["-f", "lavfi", "-i", &input, "-ac", "2", "-c:a", "mp2", "-b:a", "224k"]) else {
            eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
            continue;
        };
        let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
        let a = src.info().audio().cloned().unwrap();
        assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (rate, 2, "MPEG Audio"), "{name}");
        let want = ffmpeg_audio_f32(&ff, &f, &[]);
        let n = want.len() / 2;
        assert!(n > rate as usize * 9 / 10, "{name}: ffmpeg decoded {n} samples");
        let worst = |from: usize, len: usize| {
            let got = src.audio(from as i64, len, rate).unwrap();
            (0..len).map(|k| (0..2).map(|c| (got.channels[c][k] - want[2 * (from + k) + c]).abs()).fold(0f32, f32::max)).fold(0f32, f32::max)
        };
        let level = want.iter().fold(0f32, |m, v| m.max(v.abs()));
        assert!(level > 0.1, "{name}: the source is silent ({level})");
        // a seek into the middle first, then the whole track from the start
        let (mid, all) = (worst(n / 2 + 77, 3000), worst(0, n));
        println!("{name}: {n} samples, max error mid {mid:.2e}, whole {all:.2e}");
        assert!(mid <= 1e-4 && all <= 1e-4, "{name}: differs from ffmpeg by {mid} (mid) / {all} (whole)");
    }
}