From 0a881266c85d3e2219274afa918d7c1474a51a4e Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:34:16 -0400 Subject: [PATCH] CI: xtask ci in the dev profile, one test binary per crate, nextest (#2860) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Tests: one integration-test binary per crate Each crate with several tests/*.rs files keeps them as modules of tests/it/, so cargo links one test executable per crate instead of one per file. Files that change process-wide state or time a workload stay binaries of their own. * Profiles: line tables for workspace crates, no debug info for dependencies * Windows ARM64: nightly and on release pushes instead of every push to main * xtask ci: clippy and tests in the dev profile The release profile's thin LTO and single codegen unit made linking the test binaries take most of the gate. The codec and audio crates stay at opt-level 3 in the dev profile, so their tests run at release speed. * xtask ci: nextest, doctests, no examples, one wasm check Tests run with cargo nextest when it is installed (cargo test otherwise) and build with --lib --bins --tests, so examples aren't linked; doctests run separately. The wasm check keeps going past the first failing crate. VP8 fixture generation locks across processes, since nextest runs each test in its own. * VP9 synthetic fixtures: write aside and rename A test in another process could read a fixture ffmpeg was still writing. * Fuzz tests: a third of each sweep by default, all of it with FILMCRAFT_FUZZ_ITERS Every seed, mutation kind and thread count still runs. Header bytes each get one overwrite value and two bit flips in rotation, so every value and bit index still occurs across the region. FILMCRAFT_FUZZ_ITERS=N runs the whole sweep plus N random rounds per seed, as before. * Platform fallback tests: one failure point per hybrid path The failure-anywhere test injected a failure at four points around every restart point; the hybrid's paths differ only in the first GOP, around the second and last restart points, at the end and never. The replay test needs the first two GOPs. * Export tests: shorter ranges with the same checks The bitrate test measures 0.5–1.5 s, where each preset reaches its own target (on 0–3 s the simple demo left two presets at the same rate). The every-preset test exports three frames per preset. * Audio tests: effects in parallel, a table-driven FFT for the Opus quality metric The extreme-settings test keeps both settings and the 40 s tail for every effect, one thread each. The Opus quality metric's FFT builds its twiddles once and allocates nothing per transform (same algorithm and summation order). * CI: dev-profile gate with nextest xtask ci no longer builds with --release, so the release-profile overrides go; nextest is installed for the test step. * CI: apt packages through the cached install-apt-packages action * Review fixes: fuzz tests with a panic hook in their own binaries, loud fixture lock The fuzz tests of av1, h264, hevc, mpeg2v, vp8 and vp9 count panics through a process-wide hook, so a panic in another test of the same process would fail them; they are binaries of their own again. The VP8 fixture lock fails the test instead of skipping it silently, and the full-sweep doc comments say how many truncation points the full run samples. * Fuzz tests: rotate the default subset per run (FILMCRAFT_FUZZ_SEED replays one) * platform tests: decode-switch tests in their own binaries, one NVENC lock --- .config/nextest.toml | 14 ++++ .github/workflows/ci.yml | 20 ++--- .github/workflows/windows-arm64.yml | 13 ++- Cargo.toml | 4 + apps/filmcraft-cli/tests/{ => it}/cli.rs | 0 apps/filmcraft-cli/tests/it/main.rs | 2 + .../filmcraft-cli/tests/{ => it}/workflows.rs | 0 apps/filmcraft-web/src/download_pacing.rs | 2 +- apps/filmcraft-web/src/recovery_policy.rs | 2 +- .../tests/{ => it}/download_pacing.rs | 6 +- apps/filmcraft-web/tests/it/main.rs | 2 + .../tests/{ => it}/recovery_policy.rs | 6 +- crates/aac/tests/{ => it}/common/mod.rs | 0 crates/aac/tests/{ => it}/decoder_oracle.rs | 2 +- crates/aac/tests/{ => it}/encoder_oracle.rs | 2 +- crates/aac/tests/it/main.rs | 4 + crates/aac/tests/{ => it}/robustness.rs | 2 +- crates/apv/README.md | 4 +- crates/apv/tests/{ => it}/fuzz.rs | 0 crates/apv/tests/{ => it}/fuzz_hostile.rs | 0 crates/apv/tests/it/main.rs | 3 + crates/apv/tests/{ => it}/oracle.rs | 0 crates/audio-dsp/src/effects/mod.rs | 54 +++++++------ crates/av1/README.md | 12 +-- crates/av1/tests/{ => it}/adhoc.rs | 4 +- crates/av1/tests/{ => it}/common/mod.rs | 0 crates/av1/tests/{ => it}/conformance.rs | 4 +- crates/av1/tests/{ => it}/hostile_dims.rs | 0 crates/av1/tests/it/main.rs | 7 ++ crates/av1/tests/{ => it}/oracle_inter.rs | 2 +- crates/av1/tests/{ => it}/oracle_intra.rs | 2 +- crates/av1/tests/{ => it}/perf.rs | 4 +- crates/captions/README.md | 2 +- crates/captions/tests/it/main.rs | 3 + crates/captions/tests/{ => it}/roundtrip.rs | 0 .../tests/{ => it}/ttml_body_timing.rs | 0 .../captions/tests/{ => it}/ttml_speakers.rs | 0 .../tests/{ => it}/aac_channels_oracle.rs | 2 +- crates/codecs/tests/{ => it}/adts_file.rs | 0 crates/codecs/tests/{ => it}/common/mod.rs | 0 .../codecs/tests/{ => it}/draft_decoders.rs | 2 +- crates/codecs/tests/{ => it}/eac3_oracle.rs | 2 +- .../tests/{ => it}/frame_color_oracle.rs | 2 +- crates/codecs/tests/{ => it}/h264_draft.rs | 2 +- crates/codecs/tests/it/main.rs | 17 ++++ .../tests/{ => it}/matroska_audio_oracle.rs | 2 +- crates/codecs/tests/{ => it}/media_oracle.rs | 2 +- .../codecs/tests/{ => it}/mkv_mp1_oracle.rs | 2 +- crates/codecs/tests/{ => it}/mp4_oracle.rs | 2 +- crates/codecs/tests/{ => it}/mp4_rtmd.rs | 0 crates/codecs/tests/{ => it}/mpeg_oracle.rs | 2 +- crates/codecs/tests/{ => it}/mxf_oracle.rs | 2 +- crates/codecs/tests/{ => it}/ogg_oracle.rs | 2 +- crates/codecs/tests/{ => it}/rf64_metadata.rs | 0 crates/codecs/tests/{ => it}/y4m_file.rs | 0 crates/codecs/tests/pool_bitexact.rs | 1 + crates/dng/README.md | 4 +- crates/dng/tests/it/main.rs | 2 + crates/dng/tests/{ => it}/oracle_libraw.rs | 0 crates/dng/tests/{ => it}/roundtrip.rs | 0 crates/dnx/README.md | 8 +- crates/dnx/tests/{ => it}/common/mod.rs | 0 crates/dnx/tests/{ => it}/fuzz_hostile.rs | 73 +++++++++++++++-- crates/dnx/tests/it/main.rs | 6 ++ crates/dnx/tests/{ => it}/oracle_decode.rs | 2 +- crates/dnx/tests/{ => it}/oracle_encode.rs | 2 +- crates/dnx/tests/{ => it}/perf.rs | 2 +- crates/dnx/tests/{ => it}/roundtrip.rs | 0 crates/engine/src/export_tests.rs | 25 +++--- crates/export/tests/it/main.rs | 2 + crates/export/tests/{ => it}/pool_trim.rs | 0 .../tests/{ => it}/renderer_reentrancy.rs | 2 +- crates/h264/README.md | 6 +- crates/h264/tests/fuzz_hostile.rs | 74 +++++++++++++++-- crates/h264/tests/{ => it}/common/mod.rs | 2 +- crates/h264/tests/{ => it}/conformance.rs | 2 +- crates/h264/tests/{ => it}/headers.rs | 2 +- crates/h264/tests/it/main.rs | 6 ++ crates/h264/tests/{ => it}/perf.rs | 4 +- crates/h264/tests/{ => it}/pps_validation.rs | 0 crates/h264/tests/{ => it}/robustness.rs | 2 +- crates/h264enc/examples/h264enc_synth.rs | 2 +- crates/h264enc/tests/{ => it}/common/mod.rs | 0 .../tests/{ => it}/crf_level_and_note.rs | 2 +- .../h264enc/tests/{ => it}/ffmpeg_oracle.rs | 2 +- crates/h264enc/tests/it/main.rs | 3 + crates/hevc/README.md | 6 +- crates/hevc/tests/fuzz_hostile.rs | 74 +++++++++++++++-- crates/hevc/tests/{ => it}/common/mod.rs | 2 +- crates/hevc/tests/{ => it}/conformance.rs | 2 +- crates/hevc/tests/it/main.rs | 4 + crates/hevc/tests/{ => it}/perf.rs | 4 +- crates/hevc/tests/{ => it}/robustness.rs | 2 +- crates/interchange/README.md | 6 +- crates/interchange/tests/{ => it}/aaf_omf.rs | 2 +- crates/interchange/tests/{ => it}/api.rs | 2 +- crates/interchange/tests/{ => it}/edl.rs | 2 +- crates/interchange/tests/{ => it}/fcp7.rs | 2 +- crates/interchange/tests/{ => it}/fcpxml.rs | 2 +- .../interchange/tests/{ => it}/file_urls.rs | 0 crates/interchange/tests/it/main.rs | 12 +++ .../tests/{ => it}/media_marker_export.rs | 2 +- .../tests/{ => it}/nested_sequence_markers.rs | 2 +- crates/interchange/tests/{ => it}/otio.rs | 2 +- .../tests/{ => it}/point_markers.rs | 2 +- .../interchange/tests/{ => it}/support/mod.rs | 0 .../tests/{ => it}/uncarried_clips.rs | 2 +- crates/isobmff/README.md | 8 +- crates/isobmff/tests/{ => it}/common/mod.rs | 0 crates/isobmff/tests/{ => it}/fuzz.rs | 2 +- crates/isobmff/tests/it/main.rs | 5 ++ crates/isobmff/tests/{ => it}/oracle_demux.rs | 2 +- crates/isobmff/tests/{ => it}/oracle_mux.rs | 2 +- crates/isobmff/tests/{ => it}/roundtrip.rs | 0 crates/media/tests/{ => it}/exr_oracle.rs | 0 crates/media/tests/{ => it}/hdr_oracle.rs | 0 crates/media/tests/it/main.rs | 3 + crates/media/tests/{ => it}/tiff_ycbcr.rs | 0 crates/mpeg2v/README.md | 6 +- crates/mpeg2v/src/synth_tests.rs | 2 +- crates/mpeg2v/src/tests.rs | 2 +- crates/mpeg2v/tests/fuzz_hostile.rs | 78 +++++++++++++++--- crates/mpeg2v/tests/{ => it}/common/mod.rs | 0 crates/mpeg2v/tests/{ => it}/hostile_dims.rs | 0 crates/mpeg2v/tests/it/main.rs | 4 + crates/mpeg2v/tests/{ => it}/oracle.rs | 2 +- crates/mpeg2v/tests/{ => it}/perf.rs | 6 +- crates/mpegts/README.md | 2 +- crates/mxf/README.md | 2 +- crates/ogg/README.md | 2 +- crates/opus/tests/{ => it}/common/mod.rs | 50 +++++++++--- crates/opus/tests/it/main.rs | 4 + crates/opus/tests/{ => it}/oracle.rs | 2 +- crates/opus/tests/{ => it}/robustness.rs | 0 crates/opus/tests/{ => it}/vectors.rs | 2 +- crates/platform/README.md | 20 ++--- crates/platform/tests/{ => it}/common/mod.rs | 4 + crates/platform/tests/{ => it}/fallback.rs | 43 +++++----- .../tests/{ => it}/hardware_encode.rs | 0 crates/platform/tests/{ => it}/hevc_probe.rs | 0 crates/platform/tests/it/main.rs | 13 +++ .../tests/{ => it}/media_foundation.rs | 2 +- crates/platform/tests/{ => it}/nvenc.rs | 0 .../platform/tests/{ => it}/nvenc_export.rs | 7 +- crates/platform/tests/{ => it}/nvenc_hevc.rs | 17 ++-- .../tests/{ => it}/nvenc_hevc_export.rs | 11 +-- .../tests/{ => it}/nvenc_hevc_main10.rs | 11 +-- .../tests/{ => it}/nvenc_hevc_probe.rs | 0 .../tests/{ => it}/nvenc_rgba_input.rs | 9 +-- .../platform/tests/{ => it}/videotoolbox.rs | 2 +- crates/platform/tests/media_stack.rs | 1 + crates/platform/tests/nvdec.rs | 4 +- crates/platform/tests/setting.rs | 1 + crates/platform/tests/vaapi.rs | 4 +- crates/platform/tests/vaapi_displays.rs | 1 + crates/prores/README.md | 8 +- crates/prores/tests/{ => it}/common/mod.rs | 0 crates/prores/tests/{ => it}/fuzz.rs | 2 +- crates/prores/tests/{ => it}/fuzz_hostile.rs | 0 crates/prores/tests/it/main.rs | 7 ++ crates/prores/tests/{ => it}/oracle_decode.rs | 2 +- crates/prores/tests/{ => it}/oracle_encode.rs | 2 +- crates/prores/tests/{ => it}/perf.rs | 4 +- crates/prores/tests/{ => it}/roundtrip.rs | 2 +- crates/render/tests/it/main.rs | 4 + crates/render/tests/{ => it}/median_radius.rs | 0 .../tests/{ => it}/noise_frame_cadence.rs | 0 .../tests/{ => it}/sharpen_preview_res.rs | 0 .../tests/{ => it}/underline_direction.rs | 0 crates/ui-egui/tests/{ => it}/audio51_ui.rs | 0 .../tests/{ => it}/audio_fx_editor_ui.rs | 0 .../tests/{ => it}/caption_properties_ui.rs | 0 .../tests/{ => it}/captions_font_ui.rs | 0 crates/ui-egui/tests/{ => it}/chrome_ui.rs | 0 .../ui-egui/tests/{ => it}/clip_audio_ui.rs | 0 .../ui-egui/tests/{ => it}/clip_dialogs_ui.rs | 0 .../tests/{ => it}/clipboard_paste_ui.rs | 0 crates/ui-egui/tests/{ => it}/color_ui.rs | 0 .../{ => it}/delete_caption_tracks_ui.rs | 0 crates/ui-egui/tests/{ => it}/deselect_ui.rs | 0 .../tests/{ => it}/dialog_buttons_ui.rs | 0 .../tests/{ => it}/edit_point_menu_ui.rs | 0 .../ui-egui/tests/{ => it}/escape_drag_ui.rs | 0 .../tests/{ => it}/essential_sound_ui.rs | 0 .../ui-egui/tests/{ => it}/export_menu_ui.rs | 0 crates/ui-egui/tests/{ => it}/export_ui.rs | 0 .../ui-egui/tests/{ => it}/frame_export_ui.rs | 0 .../tests/{ => it}/graphics_handles_ui.rs | 0 .../tests/{ => it}/graphics_notice_ui.rs | 0 .../tests/{ => it}/graphics_templates_ui.rs | 0 crates/ui-egui/tests/{ => it}/graphics_ui.rs | 0 .../tests/{ => it}/interchange_export_ui.rs | 0 .../tests/{ => it}/interface_zoom_ui.rs | 0 crates/ui-egui/tests/{ => it}/keyboard_ui.rs | 0 crates/ui-egui/tests/{ => it}/keyframes_ui.rs | 0 crates/ui-egui/tests/{ => it}/links_ui.rs | 0 crates/ui-egui/tests/it/main.rs | 80 +++++++++++++++++++ .../{ => it}/marked_playback_position.rs | 0 .../tests/{ => it}/mask_roundness_ui.rs | 0 crates/ui-egui/tests/{ => it}/masks_ui.rs | 0 crates/ui-egui/tests/{ => it}/media_ui.rs | 0 crates/ui-egui/tests/{ => it}/menus_ui.rs | 0 crates/ui-egui/tests/{ => it}/meters_ui.rs | 0 crates/ui-egui/tests/{ => it}/mixer_ui.rs | 0 .../tests/{ => it}/monitor_geometry.rs | 0 crates/ui-egui/tests/{ => it}/multicam_ui.rs | 0 .../tests/{ => it}/native_dialogs_ui.rs | 0 .../ui-egui/tests/{ => it}/nest_editing_ui.rs | 0 .../tests/{ => it}/nest_rendering_ui.rs | 0 crates/ui-egui/tests/{ => it}/nested_ui.rs | 0 crates/ui-egui/tests/{ => it}/panels_ui.rs | 0 crates/ui-egui/tests/{ => it}/par_ui.rs | 0 crates/ui-egui/tests/{ => it}/perf_ui.rs | 0 .../tests/{ => it}/playback_clock_ui.rs | 0 .../tests/{ => it}/project_panel_ui.rs | 0 .../tests/{ => it}/properties_scroll_ui.rs | 0 crates/ui-egui/tests/{ => it}/remix_ui.rs | 0 crates/ui-egui/tests/{ => it}/scripted.rs | 2 +- .../tests/{ => it}/sequence_tabs_ui.rs | 0 crates/ui-egui/tests/{ => it}/sequence_ui.rs | 0 crates/ui-egui/tests/{ => it}/settings_ui.rs | 0 .../tests/{ => it}/shared_docking_ui.rs | 0 .../ui-egui/tests/{ => it}/source_drag_ui.rs | 0 .../tests/{ => it}/source_playback_ui.rs | 0 crates/ui-egui/tests/{ => it}/timecode_ui.rs | 0 .../tests/{ => it}/timeline_drop_ui.rs | 0 .../tests/{ => it}/timeline_escape_ui.rs | 0 .../ui-egui/tests/{ => it}/timeline_gap_ui.rs | 0 .../tests/{ => it}/timeline_marquee_ui.rs | 0 .../tests/{ => it}/timeline_middle_pan_ui.rs | 0 .../tests/{ => it}/timeline_option_drag_ui.rs | 0 .../tests/{ => it}/timeline_scroll_ui.rs | 0 .../tests/{ => it}/timeline_tabs_ui.rs | 0 .../{ => it}/timeline_track_height_ui.rs | 0 .../{ => it}/timeline_transition_move_ui.rs | 0 .../tests/{ => it}/timeline_trim_ui.rs | 4 +- .../ui-egui/tests/{ => it}/track_color_ui.rs | 0 .../tests/{ => it}/track_select_drag_ui.rs | 0 .../ui-egui/tests/{ => it}/transcript_ui.rs | 0 .../tests/{ => it}/transform_handles_ui.rs | 0 .../tests/{ => it}/transition_drop_ui.rs | 0 .../ui-egui/tests/{ => it}/transitions_ui.rs | 0 crates/ui-egui/tests/{ => it}/tts_ui.rs | 0 .../tests/{ => it}/ui_control_input.rs | 0 .../ui-egui/tests/{ => it}/ui_dock_resize.rs | 0 .../tests/{ => it}/ui_keyboard_focus.rs | 0 crates/ui-egui/tests/{ => it}/unlink_ui.rs | 0 crates/ui-egui/tests/{ => it}/view_ui.rs | 0 crates/ui-egui/tests/{ => it}/voiceover_ui.rs | 0 .../ui-egui/tests/{ => it}/workspaces_ui.rs | 0 crates/vp8/README.md | 4 +- crates/vp8/tests/{ => it}/common/mod.rs | 10 ++- crates/vp8/tests/{ => it}/conformance.rs | 2 +- crates/vp8/tests/it/main.rs | 4 + crates/vp8/tests/{ => it}/robustness.rs | 2 +- crates/vp8/tests/{ => it}/vectors.rs | 2 +- crates/vp9/README.md | 8 +- crates/vp9/tests/fuzz_hostile.rs | 73 +++++++++++++++-- crates/vp9/tests/{ => it}/common/mod.rs | 0 crates/vp9/tests/{ => it}/conformance.rs | 2 +- crates/vp9/tests/it/main.rs | 5 ++ crates/vp9/tests/{ => it}/perf.rs | 4 +- crates/vp9/tests/{ => it}/robustness.rs | 2 +- crates/vp9/tests/{ => it}/synthetic.rs | 9 ++- docs/contributing.md | 4 +- docs/gaps.md | 2 +- docs/keyboard.md | 2 +- docs/monitors.md | 4 +- docs/performance.md | 10 +-- docs/testing.md | 70 +++++++++------- xtask/src/main.rs | 62 ++++++++------ 271 files changed, 952 insertions(+), 373 deletions(-) create mode 100644 .config/nextest.toml rename apps/filmcraft-cli/tests/{ => it}/cli.rs (100%) create mode 100644 apps/filmcraft-cli/tests/it/main.rs rename apps/filmcraft-cli/tests/{ => it}/workflows.rs (100%) rename apps/filmcraft-web/tests/{ => it}/download_pacing.rs (87%) create mode 100644 apps/filmcraft-web/tests/it/main.rs rename apps/filmcraft-web/tests/{ => it}/recovery_policy.rs (96%) rename crates/aac/tests/{ => it}/common/mod.rs (100%) rename crates/aac/tests/{ => it}/decoder_oracle.rs (99%) rename crates/aac/tests/{ => it}/encoder_oracle.rs (99%) create mode 100644 crates/aac/tests/it/main.rs rename crates/aac/tests/{ => it}/robustness.rs (99%) rename crates/apv/tests/{ => it}/fuzz.rs (100%) rename crates/apv/tests/{ => it}/fuzz_hostile.rs (100%) create mode 100644 crates/apv/tests/it/main.rs rename crates/apv/tests/{ => it}/oracle.rs (100%) rename crates/av1/tests/{ => it}/adhoc.rs (92%) rename crates/av1/tests/{ => it}/common/mod.rs (100%) rename crates/av1/tests/{ => it}/conformance.rs (94%) rename crates/av1/tests/{ => it}/hostile_dims.rs (100%) create mode 100644 crates/av1/tests/it/main.rs rename crates/av1/tests/{ => it}/oracle_inter.rs (99%) rename crates/av1/tests/{ => it}/oracle_intra.rs (99%) rename crates/av1/tests/{ => it}/perf.rs (97%) create mode 100644 crates/captions/tests/it/main.rs rename crates/captions/tests/{ => it}/roundtrip.rs (100%) rename crates/captions/tests/{ => it}/ttml_body_timing.rs (100%) rename crates/captions/tests/{ => it}/ttml_speakers.rs (100%) rename crates/codecs/tests/{ => it}/aac_channels_oracle.rs (99%) rename crates/codecs/tests/{ => it}/adts_file.rs (100%) rename crates/codecs/tests/{ => it}/common/mod.rs (100%) rename crates/codecs/tests/{ => it}/draft_decoders.rs (99%) rename crates/codecs/tests/{ => it}/eac3_oracle.rs (99%) rename crates/codecs/tests/{ => it}/frame_color_oracle.rs (99%) rename crates/codecs/tests/{ => it}/h264_draft.rs (99%) create mode 100644 crates/codecs/tests/it/main.rs rename crates/codecs/tests/{ => it}/matroska_audio_oracle.rs (99%) rename crates/codecs/tests/{ => it}/media_oracle.rs (99%) rename crates/codecs/tests/{ => it}/mkv_mp1_oracle.rs (99%) rename crates/codecs/tests/{ => it}/mp4_oracle.rs (99%) rename crates/codecs/tests/{ => it}/mp4_rtmd.rs (100%) rename crates/codecs/tests/{ => it}/mpeg_oracle.rs (99%) rename crates/codecs/tests/{ => it}/mxf_oracle.rs (99%) rename crates/codecs/tests/{ => it}/ogg_oracle.rs (99%) rename crates/codecs/tests/{ => it}/rf64_metadata.rs (100%) rename crates/codecs/tests/{ => it}/y4m_file.rs (100%) create mode 100644 crates/dng/tests/it/main.rs rename crates/dng/tests/{ => it}/oracle_libraw.rs (100%) rename crates/dng/tests/{ => it}/roundtrip.rs (100%) rename crates/dnx/tests/{ => it}/common/mod.rs (100%) rename crates/dnx/tests/{ => it}/fuzz_hostile.rs (54%) create mode 100644 crates/dnx/tests/it/main.rs rename crates/dnx/tests/{ => it}/oracle_decode.rs (99%) rename crates/dnx/tests/{ => it}/oracle_encode.rs (99%) rename crates/dnx/tests/{ => it}/perf.rs (95%) rename crates/dnx/tests/{ => it}/roundtrip.rs (100%) create mode 100644 crates/export/tests/it/main.rs rename crates/export/tests/{ => it}/pool_trim.rs (100%) rename crates/export/tests/{ => it}/renderer_reentrancy.rs (98%) rename crates/h264/tests/{ => it}/common/mod.rs (99%) rename crates/h264/tests/{ => it}/conformance.rs (99%) rename crates/h264/tests/{ => it}/headers.rs (99%) create mode 100644 crates/h264/tests/it/main.rs rename crates/h264/tests/{ => it}/perf.rs (91%) rename crates/h264/tests/{ => it}/pps_validation.rs (100%) rename crates/h264/tests/{ => it}/robustness.rs (99%) rename crates/h264enc/tests/{ => it}/common/mod.rs (100%) rename crates/h264enc/tests/{ => it}/crf_level_and_note.rs (98%) rename crates/h264enc/tests/{ => it}/ffmpeg_oracle.rs (99%) create mode 100644 crates/h264enc/tests/it/main.rs rename crates/hevc/tests/{ => it}/common/mod.rs (99%) rename crates/hevc/tests/{ => it}/conformance.rs (99%) create mode 100644 crates/hevc/tests/it/main.rs rename crates/hevc/tests/{ => it}/perf.rs (91%) rename crates/hevc/tests/{ => it}/robustness.rs (99%) rename crates/interchange/tests/{ => it}/aaf_omf.rs (99%) rename crates/interchange/tests/{ => it}/api.rs (99%) rename crates/interchange/tests/{ => it}/edl.rs (99%) rename crates/interchange/tests/{ => it}/fcp7.rs (99%) rename crates/interchange/tests/{ => it}/fcpxml.rs (99%) rename crates/interchange/tests/{ => it}/file_urls.rs (100%) create mode 100644 crates/interchange/tests/it/main.rs rename crates/interchange/tests/{ => it}/media_marker_export.rs (99%) rename crates/interchange/tests/{ => it}/nested_sequence_markers.rs (99%) rename crates/interchange/tests/{ => it}/otio.rs (99%) rename crates/interchange/tests/{ => it}/point_markers.rs (99%) rename crates/interchange/tests/{ => it}/support/mod.rs (100%) rename crates/interchange/tests/{ => it}/uncarried_clips.rs (99%) rename crates/isobmff/tests/{ => it}/common/mod.rs (100%) rename crates/isobmff/tests/{ => it}/fuzz.rs (99%) create mode 100644 crates/isobmff/tests/it/main.rs rename crates/isobmff/tests/{ => it}/oracle_demux.rs (99%) rename crates/isobmff/tests/{ => it}/oracle_mux.rs (99%) rename crates/isobmff/tests/{ => it}/roundtrip.rs (100%) rename crates/media/tests/{ => it}/exr_oracle.rs (100%) rename crates/media/tests/{ => it}/hdr_oracle.rs (100%) create mode 100644 crates/media/tests/it/main.rs rename crates/media/tests/{ => it}/tiff_ycbcr.rs (100%) rename crates/mpeg2v/tests/{ => it}/common/mod.rs (100%) rename crates/mpeg2v/tests/{ => it}/hostile_dims.rs (100%) create mode 100644 crates/mpeg2v/tests/it/main.rs rename crates/mpeg2v/tests/{ => it}/oracle.rs (99%) rename crates/mpeg2v/tests/{ => it}/perf.rs (87%) rename crates/opus/tests/{ => it}/common/mod.rs (86%) create mode 100644 crates/opus/tests/it/main.rs rename crates/opus/tests/{ => it}/oracle.rs (99%) rename crates/opus/tests/{ => it}/robustness.rs (100%) rename crates/opus/tests/{ => it}/vectors.rs (99%) rename crates/platform/tests/{ => it}/common/mod.rs (96%) rename crates/platform/tests/{ => it}/fallback.rs (84%) rename crates/platform/tests/{ => it}/hardware_encode.rs (100%) rename crates/platform/tests/{ => it}/hevc_probe.rs (100%) create mode 100644 crates/platform/tests/it/main.rs rename crates/platform/tests/{ => it}/media_foundation.rs (99%) rename crates/platform/tests/{ => it}/nvenc.rs (100%) rename crates/platform/tests/{ => it}/nvenc_export.rs (95%) rename crates/platform/tests/{ => it}/nvenc_hevc.rs (95%) rename crates/platform/tests/{ => it}/nvenc_hevc_export.rs (97%) rename crates/platform/tests/{ => it}/nvenc_hevc_main10.rs (96%) rename crates/platform/tests/{ => it}/nvenc_hevc_probe.rs (100%) rename crates/platform/tests/{ => it}/nvenc_rgba_input.rs (94%) rename crates/platform/tests/{ => it}/videotoolbox.rs (99%) rename crates/prores/tests/{ => it}/common/mod.rs (100%) rename crates/prores/tests/{ => it}/fuzz.rs (99%) rename crates/prores/tests/{ => it}/fuzz_hostile.rs (100%) create mode 100644 crates/prores/tests/it/main.rs rename crates/prores/tests/{ => it}/oracle_decode.rs (99%) rename crates/prores/tests/{ => it}/oracle_encode.rs (99%) rename crates/prores/tests/{ => it}/perf.rs (95%) rename crates/prores/tests/{ => it}/roundtrip.rs (99%) create mode 100644 crates/render/tests/it/main.rs rename crates/render/tests/{ => it}/median_radius.rs (100%) rename crates/render/tests/{ => it}/noise_frame_cadence.rs (100%) rename crates/render/tests/{ => it}/sharpen_preview_res.rs (100%) rename crates/render/tests/{ => it}/underline_direction.rs (100%) rename crates/ui-egui/tests/{ => it}/audio51_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/audio_fx_editor_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/caption_properties_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/captions_font_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/chrome_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/clip_audio_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/clip_dialogs_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/clipboard_paste_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/color_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/delete_caption_tracks_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/deselect_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/dialog_buttons_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/edit_point_menu_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/escape_drag_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/essential_sound_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/export_menu_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/export_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/frame_export_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/graphics_handles_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/graphics_notice_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/graphics_templates_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/graphics_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/interchange_export_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/interface_zoom_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/keyboard_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/keyframes_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/links_ui.rs (100%) create mode 100644 crates/ui-egui/tests/it/main.rs rename crates/ui-egui/tests/{ => it}/marked_playback_position.rs (100%) rename crates/ui-egui/tests/{ => it}/mask_roundness_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/masks_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/media_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/menus_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/meters_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/mixer_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/monitor_geometry.rs (100%) rename crates/ui-egui/tests/{ => it}/multicam_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/native_dialogs_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/nest_editing_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/nest_rendering_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/nested_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/panels_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/par_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/perf_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/playback_clock_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/project_panel_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/properties_scroll_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/remix_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/scripted.rs (99%) rename crates/ui-egui/tests/{ => it}/sequence_tabs_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/sequence_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/settings_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/shared_docking_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/source_drag_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/source_playback_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timecode_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_drop_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_escape_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_gap_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_marquee_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_middle_pan_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_option_drag_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_scroll_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_tabs_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_track_height_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_transition_move_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/timeline_trim_ui.rs (99%) rename crates/ui-egui/tests/{ => it}/track_color_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/track_select_drag_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/transcript_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/transform_handles_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/transition_drop_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/transitions_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/tts_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/ui_control_input.rs (100%) rename crates/ui-egui/tests/{ => it}/ui_dock_resize.rs (100%) rename crates/ui-egui/tests/{ => it}/ui_keyboard_focus.rs (100%) rename crates/ui-egui/tests/{ => it}/unlink_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/view_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/voiceover_ui.rs (100%) rename crates/ui-egui/tests/{ => it}/workspaces_ui.rs (100%) rename crates/vp8/tests/{ => it}/common/mod.rs (94%) rename crates/vp8/tests/{ => it}/conformance.rs (99%) create mode 100644 crates/vp8/tests/it/main.rs rename crates/vp8/tests/{ => it}/robustness.rs (99%) rename crates/vp8/tests/{ => it}/vectors.rs (99%) rename crates/vp9/tests/{ => it}/common/mod.rs (100%) rename crates/vp9/tests/{ => it}/conformance.rs (99%) create mode 100644 crates/vp9/tests/it/main.rs rename crates/vp9/tests/{ => it}/perf.rs (92%) rename crates/vp9/tests/{ => it}/robustness.rs (99%) rename crates/vp9/tests/{ => it}/synthetic.rs (96%) diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..63e594472 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,14 @@ +# `cargo nextest run` (used by `cargo xtask ci`) runs every test in a process of its own, in parallel. + +[test-groups] +# NVENC sessions are a limited resource on consumer GPUs. +nvenc = { max-threads = 1 } + +[[profile.default.overrides]] +filter = "package(filmcraft-platform) and (binary(/nvenc/) or test(/nvenc/))" +test-group = "nvenc" + +# A throughput gate: it measures one core's speed, so nothing else runs beside it. +[[profile.default.overrides]] +filter = "package(filmcraft-render) and binary(mixer_perf)" +threads-required = "num-test-threads" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be6e31cfc..d156ff87f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,11 +31,6 @@ jobs: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: "0" RUST_BACKTRACE: "1" - # `xtask ci` builds clippy and the tests with --release. The release profile (thin LTO, one - # codegen unit) is for shipped binaries: on a 4-core runner it makes linking the ~220 test - # binaries take over an hour. CI keeps the optimisation level but builds in parallel units. - CARGO_PROFILE_RELEASE_LTO: "false" - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "16" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -51,11 +46,11 @@ jobs: # libasound2-dev is for cpal (audio playback). - name: Install Linux build dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcb1-dev libxcursor-dev \ - libxrandr-dev libxi-dev libfontconfig1-dev libfreetype-dev mesa-vulkan-drivers \ + uses: storytold/craft-common-actions/actions/install-apt-packages@main + with: + packages: >- + pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcb1-dev libxcursor-dev + libxrandr-dev libxi-dev libfontconfig1-dev libfreetype-dev mesa-vulkan-drivers libasound2-dev - name: Install Rust (stable + clippy, rustfmt, wasm32) @@ -64,6 +59,11 @@ jobs: --target wasm32-unknown-unknown rustup default stable + # `xtask ci` runs the tests with nextest when it is installed. + - uses: taiki-e/install-action@a4d3f3f761dcd95cce1b72cb61cb6d1963438177 # v2.87.30 + with: + tool: cargo-nextest + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: save-if: ${{ github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/windows-arm64.yml b/.github/workflows/windows-arm64.yml index 7666423a1..7934c954d 100644 --- a/.github/workflows/windows-arm64.yml +++ b/.github/workflows/windows-arm64.yml @@ -4,19 +4,18 @@ # out for now: PdfCraft saw an intermittent access violation in egui UI tests there.) name: Windows ARM64 +# The package build takes about 15 minutes, so it runs nightly on main and on pushes to `release` +# rather than on every push to main: a dependency or runner-image change still shows up within a +# day. PRs that change the Windows packaging itself still run it. on: + schedule: + - cron: "41 4 * * *" push: - branches: [main] + branches: [release] pull_request: paths: - - Cargo.toml - - Cargo.lock - - "**/Cargo.toml" - - "**/build.rs" - .github/workflows/windows-arm64.yml - packaging/windows/** - schedule: - - cron: "41 4 * * 1" # weekly, so drift in the runner images shows up workflow_dispatch: concurrency: diff --git a/Cargo.toml b/Cargo.toml index 341fb7ab3..752532d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,11 +92,15 @@ too_many_arguments = "allow" needless_range_loop = "allow" type_complexity = "allow" +# Line tables keep file and line in panics and backtraces at a fraction of full debug info's size; +# dependencies need none. [profile.dev.package."*"] opt-level = 2 +debug = false [profile.dev] opt-level = 1 +debug = "line-tables-only" # Codec hot loops: debug-build tests (RFC/oracle vectors) are minutes slower below opt-level 3. [profile.dev.package.filmcraft-h264] diff --git a/apps/filmcraft-cli/tests/cli.rs b/apps/filmcraft-cli/tests/it/cli.rs similarity index 100% rename from apps/filmcraft-cli/tests/cli.rs rename to apps/filmcraft-cli/tests/it/cli.rs diff --git a/apps/filmcraft-cli/tests/it/main.rs b/apps/filmcraft-cli/tests/it/main.rs new file mode 100644 index 000000000..9bdd14068 --- /dev/null +++ b/apps/filmcraft-cli/tests/it/main.rs @@ -0,0 +1,2 @@ +mod cli; +mod workflows; diff --git a/apps/filmcraft-cli/tests/workflows.rs b/apps/filmcraft-cli/tests/it/workflows.rs similarity index 100% rename from apps/filmcraft-cli/tests/workflows.rs rename to apps/filmcraft-cli/tests/it/workflows.rs diff --git a/apps/filmcraft-web/src/download_pacing.rs b/apps/filmcraft-web/src/download_pacing.rs index 01f9f4212..944e74bd3 100644 --- a/apps/filmcraft-web/src/download_pacing.rs +++ b/apps/filmcraft-web/src/download_pacing.rs @@ -1,4 +1,4 @@ -//! Spacing of browser downloads (pure, so it is tested natively in `tests/download_pacing.rs`; the +//! Spacing of browser downloads (pure, so it is tested natively in `tests/it/download_pacing.rs`; the //! anchor clicks live in [`crate::fs::download`]). //! //! Chrome drops anchor-click downloads that start in one tight burst, so an image-sequence export diff --git a/apps/filmcraft-web/src/recovery_policy.rs b/apps/filmcraft-web/src/recovery_policy.rs index 8426d3c39..7a2e0fc26 100644 --- a/apps/filmcraft-web/src/recovery_policy.rs +++ b/apps/filmcraft-web/src/recovery_policy.rs @@ -1,5 +1,5 @@ //! When the web recovery snapshot is written or marked clean (pure, so it is tested natively in -//! `tests/recovery_policy.rs`; the OPFS side lives in [`crate::recovery`]). +//! `tests/it/recovery_policy.rs`; the OPFS side lives in [`crate::recovery`]). //! //! - A dirty project is snapshotted at most every [`INTERVAL_S`] seconds, once per revision. A write //! that fails is retried after the interval, without waiting for another edit (#197). diff --git a/apps/filmcraft-web/tests/download_pacing.rs b/apps/filmcraft-web/tests/it/download_pacing.rs similarity index 87% rename from apps/filmcraft-web/tests/download_pacing.rs rename to apps/filmcraft-web/tests/it/download_pacing.rs index 893f40d3f..4f2ee8d26 100644 --- a/apps/filmcraft-web/tests/download_pacing.rs +++ b/apps/filmcraft-web/tests/it/download_pacing.rs @@ -1,10 +1,10 @@ //! Download spacing. It is pure, so it runs natively even though the rest of the crate is //! wasm32-only. -#[path = "../src/download_pacing.rs"] -mod download_pacing; +#[path = "../../src/download_pacing.rs"] +mod pacing; -use download_pacing::{GAP_MS, Pacer}; +use pacing::{GAP_MS, Pacer}; /// #378: 20 downloads requested in the same instant are spread out, not clicked in one burst. #[test] diff --git a/apps/filmcraft-web/tests/it/main.rs b/apps/filmcraft-web/tests/it/main.rs new file mode 100644 index 000000000..a223d161d --- /dev/null +++ b/apps/filmcraft-web/tests/it/main.rs @@ -0,0 +1,2 @@ +mod download_pacing; +mod recovery_policy; diff --git a/apps/filmcraft-web/tests/recovery_policy.rs b/apps/filmcraft-web/tests/it/recovery_policy.rs similarity index 96% rename from apps/filmcraft-web/tests/recovery_policy.rs rename to apps/filmcraft-web/tests/it/recovery_policy.rs index c60b34d5b..fe64599d7 100644 --- a/apps/filmcraft-web/tests/recovery_policy.rs +++ b/apps/filmcraft-web/tests/it/recovery_policy.rs @@ -1,10 +1,10 @@ //! The web recovery snapshot policy. It is pure, so it runs natively even though the rest of the //! crate is wasm32-only. -#[path = "../src/recovery_policy.rs"] -mod recovery_policy; +#[path = "../../src/recovery_policy.rs"] +mod policy; -use recovery_policy::{Action, INTERVAL_S, Policy}; +use policy::{Action, INTERVAL_S, Policy}; /// #197: a snapshot write that fails is retried after the interval without another edit. #[test] diff --git a/crates/aac/tests/common/mod.rs b/crates/aac/tests/it/common/mod.rs similarity index 100% rename from crates/aac/tests/common/mod.rs rename to crates/aac/tests/it/common/mod.rs diff --git a/crates/aac/tests/decoder_oracle.rs b/crates/aac/tests/it/decoder_oracle.rs similarity index 99% rename from crates/aac/tests/decoder_oracle.rs rename to crates/aac/tests/it/decoder_oracle.rs index 721b94486..b5591a56a 100644 --- a/crates/aac/tests/decoder_oracle.rs +++ b/crates/aac/tests/it/decoder_oracle.rs @@ -1,7 +1,7 @@ //! Decoder accuracy against ffmpeg's decoder on streams produced by ffmpeg's AAC encoder (oracle //! only; skipped when ffmpeg is absent). -mod common; +use crate::common; use common::*; diff --git a/crates/aac/tests/encoder_oracle.rs b/crates/aac/tests/it/encoder_oracle.rs similarity index 99% rename from crates/aac/tests/encoder_oracle.rs rename to crates/aac/tests/it/encoder_oracle.rs index 3cc312ace..d73abdbb8 100644 --- a/crates/aac/tests/encoder_oracle.rs +++ b/crates/aac/tests/it/encoder_oracle.rs @@ -1,6 +1,6 @@ //! Encoder quality/bitrate tests using ffmpeg as an external decoding oracle (skipped if absent). -mod common; +use crate::common; use common::*; use filmcraft_aac::{BitrateMode, EncoderConfig, WindowShape}; diff --git a/crates/aac/tests/it/main.rs b/crates/aac/tests/it/main.rs new file mode 100644 index 000000000..6f4d1e3bd --- /dev/null +++ b/crates/aac/tests/it/main.rs @@ -0,0 +1,4 @@ +mod common; +mod decoder_oracle; +mod encoder_oracle; +mod robustness; diff --git a/crates/aac/tests/robustness.rs b/crates/aac/tests/it/robustness.rs similarity index 99% rename from crates/aac/tests/robustness.rs rename to crates/aac/tests/it/robustness.rs index 9c1bb9ea0..fd0e83ee6 100644 --- a/crates/aac/tests/robustness.rs +++ b/crates/aac/tests/it/robustness.rs @@ -1,7 +1,7 @@ //! Fuzz-style robustness: mutated access units, configs and ADTS streams must never panic, and the //! encoder must accept hostile input (NaN, ±inf, > full scale, silence, odd chunk sizes). -mod common; +use crate::common; use common::*; use filmcraft_aac::{AudioSpecificConfig, Decoder, Encoder, EncoderConfig, split_adts}; diff --git a/crates/apv/README.md b/crates/apv/README.md index e84c9f5f8..906e4b4ae 100644 --- a/crates/apv/README.md +++ b/crates/apv/README.md @@ -54,7 +54,7 @@ let apvc: Vec = enc.decoder_config_record(); // APVDecoderConfigurationReco - **Metadata (§8)**: Parses `metadata_mdcv` (`payloadType == 5`, Mastering Display Color Volume) and `metadata_cll` (`payloadType == 6`, Content Light-Level Information). - **Robustness (§10)**: All size fields, tile coordinates, VLC codeword lengths, and coefficient - ranges (`-32768..=32767`) are checked before allocation or shift. Verified by `tests/fuzz.rs` + ranges (`-32768..=32767`) are checked before allocation or shift. Verified by `tests/it/fuzz.rs` across thousands of mutated and random inputs. ### Encoder @@ -67,7 +67,7 @@ let apvc: Vec = enc.decoder_config_record(); // APVDecoderConfigurationReco ## Accuracy (vs FFmpeg as an External Oracle) -`tests/oracle.rs` encodes synthetic test patterns across all 7 profiles, odd dimensions, multiple +`tests/it/oracle.rs` encodes synthetic test patterns across all 7 profiles, odd dimensions, multiple tiles, custom asymmetric quantization matrices, and HDR metadata PBUs, then decodes the resulting `.apv` streams with `ffmpeg -v error -xerror -f apv` and compares against `filmcraft_apv::decode_frame`. Because APV specifies an exact integer inverse transform, **`filmcraft-apv` is 100% bit-exact diff --git a/crates/apv/tests/fuzz.rs b/crates/apv/tests/it/fuzz.rs similarity index 100% rename from crates/apv/tests/fuzz.rs rename to crates/apv/tests/it/fuzz.rs diff --git a/crates/apv/tests/fuzz_hostile.rs b/crates/apv/tests/it/fuzz_hostile.rs similarity index 100% rename from crates/apv/tests/fuzz_hostile.rs rename to crates/apv/tests/it/fuzz_hostile.rs diff --git a/crates/apv/tests/it/main.rs b/crates/apv/tests/it/main.rs new file mode 100644 index 000000000..51c79839a --- /dev/null +++ b/crates/apv/tests/it/main.rs @@ -0,0 +1,3 @@ +mod fuzz; +mod fuzz_hostile; +mod oracle; diff --git a/crates/apv/tests/oracle.rs b/crates/apv/tests/it/oracle.rs similarity index 100% rename from crates/apv/tests/oracle.rs rename to crates/apv/tests/it/oracle.rs diff --git a/crates/audio-dsp/src/effects/mod.rs b/crates/audio-dsp/src/effects/mod.rs index 1a6f02a33..9a6f781cc 100644 --- a/crates/audio-dsp/src/effects/mod.rs +++ b/crates/audio-dsp/src/effects/mod.rs @@ -271,31 +271,37 @@ mod tests { #[test] fn extreme_settings_stay_finite_and_denormal_free() { - for e in effects() { - for setting in [0.0f32, 1.0] { - let mut fx = (e.create)(SR, 2); - for p in e.params { - fx.set_param(p.id, if setting == 0.0 { p.min } else { p.max }); - } - let mut l = noise(3, 24000, 4.0); - let mut r = noise(4, 24000, 4.0); - l[100] = 1e6; - r[200] = -1e6; - fx.process(&mut [&mut l, &mut r]); - assert!(l.iter().chain(&r).all(|v| v.is_finite()), "{} setting {setting}", e.id); - // Long silence afterwards: output decays without denormals. - fx.reset(); - let mut l = noise(3, 4800, 0.5); - let mut r = noise(4, 4800, 0.5); - fx.process(&mut [&mut l, &mut r]); - for _ in 0..40 { - let mut l = vec![0.0f32; 48000]; - let mut r = vec![0.0f32; 48000]; - fx.process(&mut [&mut l, &mut r]); - assert!(l.iter().chain(&r).all(|v| v.is_finite() && !v.is_subnormal()), "{} setting {setting}: denormal/NaN in tail", e.id); - } + // One thread per effect. The tail stays 40 s long: delays, reverbs and graphic EQs still + // produce output in its last second, some of it within 3x of the smallest normal f32. + std::thread::scope(|scope| { + for e in effects() { + scope.spawn(move || { + for setting in [0.0f32, 1.0] { + let mut fx = (e.create)(SR, 2); + for p in e.params { + fx.set_param(p.id, if setting == 0.0 { p.min } else { p.max }); + } + let mut l = noise(3, 24000, 4.0); + let mut r = noise(4, 24000, 4.0); + l[100] = 1e6; + r[200] = -1e6; + fx.process(&mut [&mut l, &mut r]); + assert!(l.iter().chain(&r).all(|v| v.is_finite()), "{} setting {setting}", e.id); + // Long silence afterwards: output decays without denormals. + fx.reset(); + let mut l = noise(3, 4800, 0.5); + let mut r = noise(4, 4800, 0.5); + fx.process(&mut [&mut l, &mut r]); + for _ in 0..40 { + let mut l = vec![0.0f32; 48000]; + let mut r = vec![0.0f32; 48000]; + fx.process(&mut [&mut l, &mut r]); + assert!(l.iter().chain(&r).all(|v| v.is_finite() && !v.is_subnormal()), "{} setting {setting}: denormal/NaN in tail", e.id); + } + } + }); } - } + }); } #[test] diff --git a/crates/av1/README.md b/crates/av1/README.md index 3045e9d37..8d29c1eeb 100644 --- a/crates/av1/README.md +++ b/crates/av1/README.md @@ -43,12 +43,12 @@ reference) is used only as an external test oracle. ## Accuracy -`tests/oracle_intra.rs` encodes all-intra streams with ffmpeg's libsvtav1 (with and without the +`tests/it/oracle_intra.rs` encodes all-intra streams with ffmpeg's libsvtav1 (with and without the in-loop filters; 64×64 and 128×128 superblocks; intra edge filter on and off) and compares every decoded frame with libdav1d: **bit-exact** on all 13 fixtures (8- and 10-bit 4:2:0; 128×128 to 1280×720 including odd sizes; testsrc2, mandelbrot, gradients and noise; presets 1–8). -`tests/conformance.rs` downloads libaom's conformance test vectors on first use +`tests/it/conformance.rs` downloads libaom's conformance test vectors on first use (storage.googleapis.com/aom-test-data) and compares with libdav1d: `av1-1-b8-02-allintra` (39 frames, deblocking + CDEF + self-guided restoration), `05-mv`, `06-mfmv`, `24-monochrome` and `b10-23-film_grain-50` are **bit-exact** in the default run. The ignored @@ -58,8 +58,8 @@ and `b10-23-film_grain-50` are **bit-exact** in the default run. The ignored with scaled inter-layer prediction), `24-monochrome` (8 and 10-bit) and `16-intra_only-intrabc-extreme-dv` (1080p intra block copy), `23-film_grain-50` (8 and 10-bit). -`tests/oracle_inter.rs` also checks draft mode: every unflagged frame stays bit-exact with -libdav1d and only flagged frames differ, with 1 and 4 threads. `tests/oracle_inter.rs` encodes SVT-AV1 GOPs (hierarchical references, compound prediction, +`tests/it/oracle_inter.rs` also checks draft mode: every unflagged frame stays bit-exact with +libdav1d and only flagged frames differ, with 1 and 4 threads. `tests/it/oracle_inter.rs` encodes SVT-AV1 GOPs (hierarchical references, compound prediction, OBMC / warped motion, motion-field projection, all loop filters; presets 3–8, 8- and 10-bit, odd sizes), plus super-resolution (denominator 12) and film-grain streams, and compares every frame with libdav1d: **bit-exact** on all 6 fixtures. @@ -76,13 +76,13 @@ independent). Results never depend on scheduling (the oracle tests compare 1 thr and libdav1d). With frame threads, pictures may be returned by a later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it. -`tests/perf.rs` (ignored) prints single-threaded fps (thread CPU time), per-stage ms/frame +`tests/it/perf.rs` (ignored) prints single-threaded fps (thread CPU time), per-stage ms/frame (`Decoder::stats()`) and all-core fps with its per-stage ms/frame for 1080p fixtures (key frames, GOPs, tiles, low delay) and a single-tile 2160p GOP; `AV1_PERF_ONLY=a,b` picks fixtures. The hot loops (sub-pixel filters, CDEF, inverse transforms, deblocking, entropy decoding) are written for auto-vectorisation. Measured 2026-10-01 on a 14-core Apple Silicon Mac (release build, SVT-AV1 preset 8 1080p -fixtures from `tests/perf.rs`), before (`31f8b3d`: hot loops only, no threading) and after tile / frame / +fixtures from `tests/it/perf.rs`), before (`31f8b3d`: hot loops only, no threading) and after tile / frame / post-filter threading. The machine was heavily loaded by other builds during both runs (load average 200–400), so the all-core wall figures understate what an idle machine reaches and are noisy; single-threaded figures use thread CPU time and are comparable. diff --git a/crates/av1/tests/adhoc.rs b/crates/av1/tests/it/adhoc.rs similarity index 92% rename from crates/av1/tests/adhoc.rs rename to crates/av1/tests/it/adhoc.rs index e140b9768..1595bc257 100644 --- a/crates/av1/tests/adhoc.rs +++ b/crates/av1/tests/it/adhoc.rs @@ -1,7 +1,7 @@ //! Compare an arbitrary IVF file with libdav1d: -//! `AV1_IVF=/path/x.ivf AV1_PIX=yuv420p cargo test -p filmcraft-av1 --test adhoc -- --ignored --nocapture` +//! `AV1_IVF=/path/x.ivf AV1_PIX=yuv420p cargo test -p filmcraft-av1 --test it adhoc:: -- --ignored --nocapture` -mod common; +use crate::common; use common::*; #[test] diff --git a/crates/av1/tests/common/mod.rs b/crates/av1/tests/it/common/mod.rs similarity index 100% rename from crates/av1/tests/common/mod.rs rename to crates/av1/tests/it/common/mod.rs diff --git a/crates/av1/tests/conformance.rs b/crates/av1/tests/it/conformance.rs similarity index 94% rename from crates/av1/tests/conformance.rs rename to crates/av1/tests/it/conformance.rs index 033f590e2..b70270af0 100644 --- a/crates/av1/tests/conformance.rs +++ b/crates/av1/tests/it/conformance.rs @@ -1,6 +1,6 @@ //! libaom conformance test vectors (downloaded on first use), compared with libdav1d. -mod common; +use crate::common; use common::*; /// Vectors the decoder handles today (more are added as stages land). @@ -17,7 +17,7 @@ fn conformance_vectors() { } /// The wider libaom set (slow: many downloads and decodes): -/// `cargo test -p filmcraft-av1 --test conformance -- --ignored`; `AV1_VECTORS=a,b` overrides it. +/// `cargo test -p filmcraft-av1 --test it conformance:: -- --ignored`; `AV1_VECTORS=a,b` overrides it. #[test] #[ignore] fn conformance_vectors_extended() { diff --git a/crates/av1/tests/hostile_dims.rs b/crates/av1/tests/it/hostile_dims.rs similarity index 100% rename from crates/av1/tests/hostile_dims.rs rename to crates/av1/tests/it/hostile_dims.rs diff --git a/crates/av1/tests/it/main.rs b/crates/av1/tests/it/main.rs new file mode 100644 index 000000000..decf05989 --- /dev/null +++ b/crates/av1/tests/it/main.rs @@ -0,0 +1,7 @@ +mod adhoc; +mod common; +mod conformance; +mod hostile_dims; +mod oracle_inter; +mod oracle_intra; +mod perf; diff --git a/crates/av1/tests/oracle_inter.rs b/crates/av1/tests/it/oracle_inter.rs similarity index 99% rename from crates/av1/tests/oracle_inter.rs rename to crates/av1/tests/it/oracle_inter.rs index 06a4d88fe..5d443194f 100644 --- a/crates/av1/tests/oracle_inter.rs +++ b/crates/av1/tests/it/oracle_inter.rs @@ -2,7 +2,7 @@ //! GOPs with hierarchical references, compound prediction, OBMC / warped motion, global motion, //! motion-field projection and all loop filters. -mod common; +use crate::common; use common::*; fn specs() -> Vec { diff --git a/crates/av1/tests/oracle_intra.rs b/crates/av1/tests/it/oracle_intra.rs similarity index 99% rename from crates/av1/tests/oracle_intra.rs rename to crates/av1/tests/it/oracle_intra.rs index 441220f18..d002af67e 100644 --- a/crates/av1/tests/oracle_intra.rs +++ b/crates/av1/tests/it/oracle_intra.rs @@ -1,6 +1,6 @@ //! Intra-frame bit-exactness against libdav1d (run as an external ffmpeg process). -mod common; +use crate::common; use common::*; const NO_FILTERS: &str = "enable-cdef=0:enable-restoration=0:enable-dlf=0:keyint=1"; diff --git a/crates/av1/tests/perf.rs b/crates/av1/tests/it/perf.rs similarity index 97% rename from crates/av1/tests/perf.rs rename to crates/av1/tests/it/perf.rs index 9047605aa..518a0cac3 100644 --- a/crates/av1/tests/perf.rs +++ b/crates/av1/tests/it/perf.rs @@ -1,11 +1,11 @@ //! Decode throughput and per-stage breakdown (ignored; run with -//! `cargo test --release -p filmcraft-av1 --test perf -- --ignored --nocapture`). +//! `cargo test --release -p filmcraft-av1 --test it perf:: -- --ignored --nocapture`). //! //! The machine may be busy (other builds), so single-threaded work is also measured with the //! thread CPU clock, which is robust to load; wall-clock figures are printed next to the load //! average. -mod common; +use crate::common; use common::*; use filmcraft_av1::{DecodeStats, Decoder, Stage}; use std::time::Instant; diff --git a/crates/captions/README.md b/crates/captions/README.md index 04183352c..8618cadac 100644 --- a/crates/captions/README.md +++ b/crates/captions/README.md @@ -85,7 +85,7 @@ implementation's code was consulted. - Unit tests per module (timestamps, decoding, sloppy SRT, WebVTT blocks and voices, CEA-608 tables, PAC round trips, known SCC pop-on and roll-up streams, layout). -- `tests/roundtrip.rs`: property tests — SRT (also with BOM + CRLF), WebVTT (ids, speakers, +- `tests/it/roundtrip.rs`: property tests — SRT (also with BOM + CRLF), WebVTT (ids, speakers, settings, STYLE blocks), SCC and MCC (drop-frame and non-drop, gaps and back-to-back captions; MCC also through CEA-708 alone), EBU STL (25 and 29.97 fps, accents, extension blocks), TTML (23.976 / 25 / 29.97 / 59.94, ids, speakers) round-trip frame-exactly; DFXP round-trips after diff --git a/crates/captions/tests/it/main.rs b/crates/captions/tests/it/main.rs new file mode 100644 index 000000000..d890ff913 --- /dev/null +++ b/crates/captions/tests/it/main.rs @@ -0,0 +1,3 @@ +mod roundtrip; +mod ttml_body_timing; +mod ttml_speakers; diff --git a/crates/captions/tests/roundtrip.rs b/crates/captions/tests/it/roundtrip.rs similarity index 100% rename from crates/captions/tests/roundtrip.rs rename to crates/captions/tests/it/roundtrip.rs diff --git a/crates/captions/tests/ttml_body_timing.rs b/crates/captions/tests/it/ttml_body_timing.rs similarity index 100% rename from crates/captions/tests/ttml_body_timing.rs rename to crates/captions/tests/it/ttml_body_timing.rs diff --git a/crates/captions/tests/ttml_speakers.rs b/crates/captions/tests/it/ttml_speakers.rs similarity index 100% rename from crates/captions/tests/ttml_speakers.rs rename to crates/captions/tests/it/ttml_speakers.rs diff --git a/crates/codecs/tests/aac_channels_oracle.rs b/crates/codecs/tests/it/aac_channels_oracle.rs similarity index 99% rename from crates/codecs/tests/aac_channels_oracle.rs rename to crates/codecs/tests/it/aac_channels_oracle.rs index 5f850ac3e..43bcf6a65 100644 --- a/crates/codecs/tests/aac_channels_oracle.rs +++ b/crates/codecs/tests/it/aac_channels_oracle.rs @@ -3,7 +3,7 @@ //! both 7.1 (eight channels) and 11 is 6.1 (seven). Each channel carries its own tone, so a //! dropped or swapped channel shows up against ffmpeg's decode. -mod common; +use crate::common; use std::io::Cursor; diff --git a/crates/codecs/tests/adts_file.rs b/crates/codecs/tests/it/adts_file.rs similarity index 100% rename from crates/codecs/tests/adts_file.rs rename to crates/codecs/tests/it/adts_file.rs diff --git a/crates/codecs/tests/common/mod.rs b/crates/codecs/tests/it/common/mod.rs similarity index 100% rename from crates/codecs/tests/common/mod.rs rename to crates/codecs/tests/it/common/mod.rs diff --git a/crates/codecs/tests/draft_decoders.rs b/crates/codecs/tests/it/draft_decoders.rs similarity index 99% rename from crates/codecs/tests/draft_decoders.rs rename to crates/codecs/tests/it/draft_decoders.rs index bec6476f9..b656dea50 100644 --- a/crates/codecs/tests/draft_decoders.rs +++ b/crates/codecs/tests/it/draft_decoders.rs @@ -2,7 +2,7 @@ //! off by default, approximate frames only reach draft requests, and decoding for an export after //! draft playback (same source, same GOP cache) still matches ffmpeg bit for bit. -mod common; +use crate::common; use common::*; use filmcraft_media::FrameRequest; diff --git a/crates/codecs/tests/eac3_oracle.rs b/crates/codecs/tests/it/eac3_oracle.rs similarity index 99% rename from crates/codecs/tests/eac3_oracle.rs rename to crates/codecs/tests/it/eac3_oracle.rs index 7d875d49c..e75476db7 100644 --- a/crates/codecs/tests/eac3_oracle.rs +++ b/crates/codecs/tests/it/eac3_oracle.rs @@ -2,7 +2,7 @@ //! and muxes the files and decodes them as the oracle. Zero-bit mantissas carry decoder-specific //! dither (A/52 §7.3.4), so the criterion is SNR against ffmpeg's decode, sample-aligned. -mod common; +use crate::common; use common::*; diff --git a/crates/codecs/tests/frame_color_oracle.rs b/crates/codecs/tests/it/frame_color_oracle.rs similarity index 99% rename from crates/codecs/tests/frame_color_oracle.rs rename to crates/codecs/tests/it/frame_color_oracle.rs index 7da3baf09..15740fc3e 100644 --- a/crates/codecs/tests/frame_color_oracle.rs +++ b/crates/codecs/tests/it/frame_color_oracle.rs @@ -2,7 +2,7 @@ //! MP4 without a `colr` box unless asked, so for most files the bitstream is the only place the //! primaries are described: stream info already read them, the decoded frames reported BT.709. -mod common; +use crate::common; use common::*; use filmcraft_color::{ColorInfo, ColorSpace, Matrix, Primaries, Range, Transfer}; diff --git a/crates/codecs/tests/h264_draft.rs b/crates/codecs/tests/it/h264_draft.rs similarity index 99% rename from crates/codecs/tests/h264_draft.rs rename to crates/codecs/tests/it/h264_draft.rs index 81e8c3bb8..82721ee30 100644 --- a/crates/codecs/tests/h264_draft.rs +++ b/crates/codecs/tests/it/h264_draft.rs @@ -2,7 +2,7 @@ //! by default, its approximate frames only ever reach draft requests, and decoding for an export //! after draft playback (same source, same GOP cache) still matches ffmpeg bit for bit. -mod common; +use crate::common; use common::*; use filmcraft_media::FrameRequest; diff --git a/crates/codecs/tests/it/main.rs b/crates/codecs/tests/it/main.rs new file mode 100644 index 000000000..679dcaa6b --- /dev/null +++ b/crates/codecs/tests/it/main.rs @@ -0,0 +1,17 @@ +mod aac_channels_oracle; +mod adts_file; +mod common; +mod draft_decoders; +mod eac3_oracle; +mod frame_color_oracle; +mod h264_draft; +mod matroska_audio_oracle; +mod media_oracle; +mod mkv_mp1_oracle; +mod mp4_oracle; +mod mp4_rtmd; +mod mpeg_oracle; +mod mxf_oracle; +mod ogg_oracle; +mod rf64_metadata; +mod y4m_file; diff --git a/crates/codecs/tests/matroska_audio_oracle.rs b/crates/codecs/tests/it/matroska_audio_oracle.rs similarity index 99% rename from crates/codecs/tests/matroska_audio_oracle.rs rename to crates/codecs/tests/it/matroska_audio_oracle.rs index 3ded7af07..c2cfbd45e 100644 --- a/crates/codecs/tests/matroska_audio_oracle.rs +++ b/crates/codecs/tests/it/matroska_audio_oracle.rs @@ -4,7 +4,7 @@ //! reads it, a tone must stay as smooth as ffmpeg's decode of it: no holes or overlaps between //! packets. -mod common; +use crate::common; use common::*; diff --git a/crates/codecs/tests/media_oracle.rs b/crates/codecs/tests/it/media_oracle.rs similarity index 99% rename from crates/codecs/tests/media_oracle.rs rename to crates/codecs/tests/it/media_oracle.rs index da95c5eb6..5091c1dc3 100644 --- a/crates/codecs/tests/media_oracle.rs +++ b/crates/codecs/tests/it/media_oracle.rs @@ -4,7 +4,7 @@ //! exactly, JPEG within a few levels, high bit depths within one 8-bit step), and the BWF //! TimeReference must give the start timecode. -mod common; +use crate::common; use std::path::Path; use std::sync::Arc; diff --git a/crates/codecs/tests/mkv_mp1_oracle.rs b/crates/codecs/tests/it/mkv_mp1_oracle.rs similarity index 99% rename from crates/codecs/tests/mkv_mp1_oracle.rs rename to crates/codecs/tests/it/mkv_mp1_oracle.rs index 05d9810fb..aa142457c 100644 --- a/crates/codecs/tests/mkv_mp1_oracle.rs +++ b/crates/codecs/tests/it/mkv_mp1_oracle.rs @@ -2,7 +2,7 @@ //! installed encoder writes Layer I, so the frames are synthetic (valid random Layer I frames, //! as in `crates/mpa/tests/oracle.rs`) and ffmpeg only remuxes them into Matroska and decodes. -mod common; +use crate::common; use common::*; diff --git a/crates/codecs/tests/mp4_oracle.rs b/crates/codecs/tests/it/mp4_oracle.rs similarity index 99% rename from crates/codecs/tests/mp4_oracle.rs rename to crates/codecs/tests/it/mp4_oracle.rs index cbb71ba0e..892598daf 100644 --- a/crates/codecs/tests/mp4_oracle.rs +++ b/crates/codecs/tests/it/mp4_oracle.rs @@ -2,7 +2,7 @@ //! priming), and reads at another sample rate must land on the same source samples as the native //! decode, however the requests are cut. -mod common; +use crate::common; use common::*; diff --git a/crates/codecs/tests/mp4_rtmd.rs b/crates/codecs/tests/it/mp4_rtmd.rs similarity index 100% rename from crates/codecs/tests/mp4_rtmd.rs rename to crates/codecs/tests/it/mp4_rtmd.rs diff --git a/crates/codecs/tests/mpeg_oracle.rs b/crates/codecs/tests/it/mpeg_oracle.rs similarity index 99% rename from crates/codecs/tests/mpeg_oracle.rs rename to crates/codecs/tests/it/mpeg_oracle.rs index e6d5a39dd..bf58294e8 100644 --- a/crates/codecs/tests/mpeg_oracle.rs +++ b/crates/codecs/tests/it/mpeg_oracle.rs @@ -4,7 +4,7 @@ //! random seeks give the frames of the sequential decode. Audio: LPCM sample-exact, MPEG audio //! and AAC within a float-decoder tolerance, starting at the same sample. -mod common; +use crate::common; use std::path::{Path, PathBuf}; diff --git a/crates/codecs/tests/mxf_oracle.rs b/crates/codecs/tests/it/mxf_oracle.rs similarity index 99% rename from crates/codecs/tests/mxf_oracle.rs rename to crates/codecs/tests/it/mxf_oracle.rs index 5a965b814..23d882755 100644 --- a/crates/codecs/tests/mxf_oracle.rs +++ b/crates/codecs/tests/it/mxf_oracle.rs @@ -2,7 +2,7 @@ //! oracle. Our decode must match frame for frame (H.264 bit-exact, VC-3 ±2, ProRes ±1 LSB), seek //! to random frames exactly, and give sample-exact PCM. -mod common; +use crate::common; use std::path::{Path, PathBuf}; diff --git a/crates/codecs/tests/ogg_oracle.rs b/crates/codecs/tests/it/ogg_oracle.rs similarity index 99% rename from crates/codecs/tests/ogg_oracle.rs rename to crates/codecs/tests/it/ogg_oracle.rs index 939407756..be02870ed 100644 --- a/crates/codecs/tests/ogg_oracle.rs +++ b/crates/codecs/tests/it/ogg_oracle.rs @@ -4,7 +4,7 @@ //! `filmcraft-opus`'s own oracle; Vorbis: SNR ≥ 60 dB), and any random window must equal the //! continuous decode (granule-position seeking with pre-roll). -mod common; +use crate::common; use std::path::{Path, PathBuf}; diff --git a/crates/codecs/tests/rf64_metadata.rs b/crates/codecs/tests/it/rf64_metadata.rs similarity index 100% rename from crates/codecs/tests/rf64_metadata.rs rename to crates/codecs/tests/it/rf64_metadata.rs diff --git a/crates/codecs/tests/y4m_file.rs b/crates/codecs/tests/it/y4m_file.rs similarity index 100% rename from crates/codecs/tests/y4m_file.rs rename to crates/codecs/tests/it/y4m_file.rs diff --git a/crates/codecs/tests/pool_bitexact.rs b/crates/codecs/tests/pool_bitexact.rs index a90ec658e..1261e9a2b 100644 --- a/crates/codecs/tests/pool_bitexact.rs +++ b/crates/codecs/tests/pool_bitexact.rs @@ -3,6 +3,7 @@ //! primed with buffers full of 0xAA before decoding, and every frame must still match ffmpeg bit //! for bit. (Its own process: the pool is global.) +#[path = "it/common/mod.rs"] mod common; use std::sync::Arc; diff --git a/crates/dng/README.md b/crates/dng/README.md index 4c2b1e7e9..c8678c1d3 100644 --- a/crates/dng/README.md +++ b/crates/dng/README.md @@ -73,14 +73,14 @@ restart intervals that split a line, and MXF-wrapped CinemaDNG. ## Tests -- `tests/roundtrip.rs`: container parsing in both byte orders with and without a preview IFD; +- `tests/it/roundtrip.rs`: container parsing in both byte orders with and without a preview IFD; uncompressed samples bit-exact at 8–16 bits, strips and tiles; lossless JPEG bit-exact for every predictor and component count, full-swing 16-bit data (every difference category, SSSS = 16), restarts and point transform; developed values against hand-computed ones (black / white, white balance, ForwardMatrix, BaselineExposure, ActiveArea, DefaultCrop, LinearRaw); CinemaDNG FrameRate / TimeCodes / ReelName; hostile files (every truncation, 12 000 mutations under `catch_unwind`, cyclic IFDs, huge sizes, bad offsets, zero denominators). -- `tests/oracle_libraw.rs`: LibRaw's `unprocessed_raw` reads the same samples from our synthetic +- `tests/it/oracle_libraw.rs`: LibRaw's `unprocessed_raw` reads the same samples from our synthetic files (uncompressed 8–16-bit in both byte orders, lossless JPEG predictors 1–7 with one and two components). Skipped when it is not installed. - Unit tests: matrix maths, the published sRGB matrix, McCamy temperatures, calibration diff --git a/crates/dng/tests/it/main.rs b/crates/dng/tests/it/main.rs new file mode 100644 index 000000000..fd498d961 --- /dev/null +++ b/crates/dng/tests/it/main.rs @@ -0,0 +1,2 @@ +mod oracle_libraw; +mod roundtrip; diff --git a/crates/dng/tests/oracle_libraw.rs b/crates/dng/tests/it/oracle_libraw.rs similarity index 100% rename from crates/dng/tests/oracle_libraw.rs rename to crates/dng/tests/it/oracle_libraw.rs diff --git a/crates/dng/tests/roundtrip.rs b/crates/dng/tests/it/roundtrip.rs similarity index 100% rename from crates/dng/tests/roundtrip.rs rename to crates/dng/tests/it/roundtrip.rs diff --git a/crates/dnx/README.md b/crates/dnx/README.md index 596ea9fa2..9f4656817 100644 --- a/crates/dnx/README.md +++ b/crates/dnx/README.md @@ -59,7 +59,7 @@ first frame header to label the stream. 12-bit coefficient domain is ¼ of the orthonormal DCT, as Table 14's 13-bit IDCT input range for 12-bit samples implies) and 1 otherwise. - Robustness: every offset and size is validated, run lengths past coefficient 63 are rejected, - and corrupted or truncated streams return `Err` without panicking (`tests/roundtrip.rs`). + and corrupted or truncated streams return `Err` without panicking (`tests/it/roundtrip.rs`). ### Where we follow ffmpeg rather than the letter of the standard @@ -100,7 +100,7 @@ the prose of §7.3.1.2; we follow the prose (normative text takes precedence). ## Accuracy (vs ffmpeg as an external oracle) -`tests/oracle_decode.rs` decodes 22 fixtures made by ffmpeg's VC-3 encoder: CIDs 1235, 1237, +`tests/it/oracle_decode.rs` decodes 22 fixtures made by ffmpeg's VC-3 encoder: CIDs 1235, 1237, 1238, 1241, 1242, 1243, 1244, 1250, 1251, 1252, 1253, 1258, 1259 and DNxHR LB/SQ/HQ/HQX/444 (Y′CbCr and RGB), with testsrc2, mandelbrot and heavy noise, at 1920×1080, 1280×720, 960×720, 1440×1080 (thin raster, interlaced), 1000×562, 640×360, 256×120 and 3840×2160. Compared with @@ -111,7 +111,7 @@ decoder evaluates the exact IDCT of equation 8.3, so decode is **not bit-exact, precision** of ffmpeg. (ffmpeg wraps rather than clips 10-bit reconstructions that overflow 1023, so the noise fixtures keep their samples inside 24…231 before encoding.) -`tests/oracle_encode.rs` encodes 1920×1080 frames with every profile and writes them into a +`tests/it/oracle_encode.rs` encodes 1920×1080 frames with every profile and writes them into a minimal test-only MOV. ffmpeg decodes them with `-xerror`, without errors, and agrees with our decoder within ±2 (90–99% identical). Luma PSNR against the source: HQ 58.8 dB (mandelbrot) and 66.3 dB (testsrc2), SQ 57.5 dB, LB 56.1 dB, HQX 10-bit 67.5 dB, HQX 12-bit 80.2 dB, 444 10-bit @@ -119,7 +119,7 @@ decoder within ±2 (90–99% identical). Luma PSNR against the source: HQ 58.8 d ## Performance -`cargo test --release -p filmcraft-dnx --test perf -- --ignored --nocapture` on an M4 Pro +`cargo test --release -p filmcraft-dnx --test it perf:: -- --ignored --nocapture` on an M4 Pro (14 cores) while the machine was **heavily overloaded** by other jobs (load average ≈ 190). Under the same load `filmcraft-prores`' perf test measured 30 fps threaded / 11.8 fps single for a 1080p HQ frame, against 318 / 53 fps in its README on a less loaded machine, so these are diff --git a/crates/dnx/tests/common/mod.rs b/crates/dnx/tests/it/common/mod.rs similarity index 100% rename from crates/dnx/tests/common/mod.rs rename to crates/dnx/tests/it/common/mod.rs diff --git a/crates/dnx/tests/fuzz_hostile.rs b/crates/dnx/tests/it/fuzz_hostile.rs similarity index 54% rename from crates/dnx/tests/fuzz_hostile.rs rename to crates/dnx/tests/it/fuzz_hostile.rs index b3eee0037..8ff02e27c 100644 --- a/crates/dnx/tests/fuzz_hostile.rs +++ b/crates/dnx/tests/it/fuzz_hostile.rs @@ -58,27 +58,34 @@ fn run(d: &[u8], label: &str) { /// Seconds the whole sweep may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). const DEADLINE_SECS: u64 = 600; +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=1500` runs all of it (see [`Depth`]). #[test] fn hostile_frames_never_panic() { filmcraft_testkit::with_deadline(DEADLINE_SECS, sweep); } fn sweep() { + let depth = Depth::from_env(); let seeds = seeds(); assert!(!seeds.is_empty()); for (si, seed) in seeds.iter().enumerate() { - let stride = (seed.len() / 400).max(1); - for n in (0..seed.len()).step_by(stride) { + for n in depth.truncations(seed.len()) { run(&seed[..n], &format!("seed {si} truncate {n}")); } // The header (and the start of the scan-line index): every byte to extreme values, every bit. for p in 0..seed.len().min(0x300) { - for v in [0u8, 0xff, 0x80, 0x01, 0x7f] { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(p, i) { + continue; + } let mut d = seed.clone(); d[p] = v; run(&d, &format!("seed {si} byte {p:#x}={v:#x}")); } for b in 0..8 { + if !depth.bit(p, b) { + continue; + } let mut d = seed.clone(); d[p] ^= 1 << b; run(&d, &format!("seed {si} flip {p:#x}.{b}")); @@ -93,8 +100,8 @@ fn sweep() { run(&d, &format!("seed {si} u16 {off:#x}={v:#x}")); } let mut rng = Rng(0x9e3779b97f4a7c15 ^ si as u64); - let iters = std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()).unwrap_or(1500); - for i in 0..iters { + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { let mut d = seed.clone(); for _ in 0..1 + rng.below(8) { let p = rng.below(d.len()); @@ -104,7 +111,61 @@ fn sweep() { d[p] = rng.next() as u8; } } - run(&d, &format!("seed {si} random {i}")); + if depth.round(i) { + run(&d, &format!("seed {si} random {i}")); + } } } } + +const VALUES: [u8; 5] = [0, 0xff, 0x80, 0x01, 0x7f]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 400 truncation +/// points spread over each seed, every header byte, overwrite value and bit flip) plus N random +/// rounds per seed: 1500 is the full depth, more hunts longer. Unset, CI runs a third of the +/// truncation points and of the full depth's random rounds and, for each header byte, one value +/// and two bits. A rotation offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC +/// day number), so every case of the full sweep runs within a few days, and a random round's +/// mutations match the same round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 400, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=1500 runs everything)"); + Depth { full: false, truncations: 400, rounds: 1500, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, p: usize, i: usize) -> bool { + self.full || i == (p + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, p: usize, b: usize) -> bool { + self.full || (b + p + self.rotation % 4).is_multiple_of(4) + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) +} diff --git a/crates/dnx/tests/it/main.rs b/crates/dnx/tests/it/main.rs new file mode 100644 index 000000000..c12ebd099 --- /dev/null +++ b/crates/dnx/tests/it/main.rs @@ -0,0 +1,6 @@ +mod common; +mod fuzz_hostile; +mod oracle_decode; +mod oracle_encode; +mod perf; +mod roundtrip; diff --git a/crates/dnx/tests/oracle_decode.rs b/crates/dnx/tests/it/oracle_decode.rs similarity index 99% rename from crates/dnx/tests/oracle_decode.rs rename to crates/dnx/tests/it/oracle_decode.rs index 476ee68d1..a9da675c8 100644 --- a/crates/dnx/tests/oracle_decode.rs +++ b/crates/dnx/tests/it/oracle_decode.rs @@ -1,6 +1,6 @@ //! Decode ffmpeg-made DNxHD / DNxHR fixtures and compare with ffmpeg's own decode. -mod common; +use crate::common; use common::*; use filmcraft_dnx::{DecodeOptions, decode_frame_with}; diff --git a/crates/dnx/tests/oracle_encode.rs b/crates/dnx/tests/it/oracle_encode.rs similarity index 99% rename from crates/dnx/tests/oracle_encode.rs rename to crates/dnx/tests/it/oracle_encode.rs index ca64941df..4d5702698 100644 --- a/crates/dnx/tests/oracle_encode.rs +++ b/crates/dnx/tests/it/oracle_encode.rs @@ -1,6 +1,6 @@ //! Encode DNxHR with our encoder, decode with ffmpeg (`-xerror`) and with our decoder. -mod common; +use crate::common; use common::*; use filmcraft_dnx::{ChromaFormat, Encoder, EncoderConfig, Profile, decode_frame}; diff --git a/crates/dnx/tests/perf.rs b/crates/dnx/tests/it/perf.rs similarity index 95% rename from crates/dnx/tests/perf.rs rename to crates/dnx/tests/it/perf.rs index 255f1961c..befd20a48 100644 --- a/crates/dnx/tests/perf.rs +++ b/crates/dnx/tests/it/perf.rs @@ -1,4 +1,4 @@ -//! Throughput: `cargo test --release -p filmcraft-dnx --test perf -- --ignored --nocapture`. +//! Throughput: `cargo test --release -p filmcraft-dnx --test it perf:: -- --ignored --nocapture`. use filmcraft_dnx::{ChromaFormat, DecodeOptions, Encoder, EncoderConfig, Frame, Profile, decode_frame_with}; use std::time::Instant; diff --git a/crates/dnx/tests/roundtrip.rs b/crates/dnx/tests/it/roundtrip.rs similarity index 100% rename from crates/dnx/tests/roundtrip.rs rename to crates/dnx/tests/it/roundtrip.rs diff --git a/crates/engine/src/export_tests.rs b/crates/engine/src/export_tests.rs index 2e6eb8a99..57a07791c 100644 --- a/crates/engine/src/export_tests.rs +++ b/crates/engine/src/export_tests.rs @@ -155,11 +155,13 @@ fn files_in(dir: &Path) -> Vec { v } -/// Every built-in preset: a quarter second of the demo sequence (seven frames at 23.976) exported +/// Every built-in preset: 1.0 to 1.08 s of the demo sequence (three frames at 23.976) exported /// with `file.exportMedia {preset}`; ffprobe checks container, codec, profile, frame size, frame -/// rate and audio format, and the data rate where the codec fixes it. +/// rate and audio format, and the data rate where the codec fixes it. Three frames are enough for +/// all of these, and each extra frame costs a 4K H.264 encode in five presets. #[test] fn every_builtin_preset_exports_and_ffprobe_confirms() { + let (start, end) = (1.0, 1.08); let mut s = demo(); let presets = s.execute("export.presets.list", json!({})).unwrap()["presets"].as_array().unwrap().clone(); assert!(presets.len() >= 24); @@ -169,7 +171,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { let sub = dir.0.join(format!("p{}", filmcraft_export::presets::preset_key(name))); std::fs::create_dir_all(&sub).unwrap(); let path = sub.join("Demo").to_string_lossy().to_string(); - let params = json!({"preset": name, "path": path, "range": "custom", "startSeconds": 1.0, "endSeconds": 1.25, "wait": true}); + let params = json!({"preset": name, "path": path, "range": "custom", "startSeconds": start, "endSeconds": end, "wait": true}); let frames = s.execute("export.resolve", params.clone()).unwrap()["frames"].as_u64().unwrap(); let r = s.execute("file.exportMedia", params).unwrap_or_else(|e| panic!("{name}: {e}")); let out = r["path"].as_str().unwrap().to_string(); @@ -182,7 +184,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { if matches!(fmt.as_str(), "png" | "tiff" | "bmp" | "jpg" | "tga" | "dpx" | "exr") { let files = files_in(&sub); let ext = Path::new(&out).extension().unwrap().to_string_lossy().to_string(); - assert_eq!(frames, 7, "frames 23..=29 at 23.976 fps"); + assert_eq!(frames, 3, "frames 23..=25 at 23.976 fps"); let want: Vec = (0..frames).map(|i| format!("Demo{i:03}.{ext}")).collect(); assert_eq!(files, want, "{name}"); if let Some(j) = probe(&sub.join(&want[0]).to_string_lossy()) { @@ -218,7 +220,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { assert_eq!(a["codec_name"], codec, "{name}"); assert_eq!(a["sample_rate"], "48000", "{name}"); assert_eq!(a["channels"], 2, "{name}"); - assert!((num(&j["format"]["duration"]) - 0.25).abs() < 0.01, "{name}: {}", j["format"]["duration"]); + assert!((num(&j["format"]["duration"]) - (end - start)).abs() < 0.01, "{name}: {}", j["format"]["duration"]); continue; } "aac" => { @@ -228,7 +230,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { assert_eq!(j["streams"].as_array().unwrap().len(), 1, "{name}"); let a = stream(&j, "audio").unwrap(); assert_eq!((a["codec_name"].as_str(), a["sample_rate"].as_str(), a["channels"].as_u64()), (Some("aac"), Some("48000"), Some(2)), "{name}"); - assert!((num(&j["format"]["duration"]) - 0.25).abs() < 0.03, "{name}: {}", j["format"]["duration"]); + assert!((num(&j["format"]["duration"]) - (end - start)).abs() < 0.03, "{name}: {}", j["format"]["duration"]); continue; } "gif" => { @@ -332,9 +334,6 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { } } -/// H.264 bitrate control over three seconds of the demo: the adaptive Match Source preset -/// targets width × height × fps × bpp, the 1080p delivery presets their fixed targets; ffprobe's -/// measured video bitrate stays within the VBR maximum and close to the target. #[test] fn aac_audio_keeps_a_dot_aac_path_as_adts_and_adds_m4a_otherwise() { let mut s = demo(); @@ -354,6 +353,12 @@ fn aac_audio_keeps_a_dot_aac_path_as_adts_and_adds_m4a_otherwise() { assert_eq!(&std::fs::read(&m4a).unwrap()[4..8], b"ftyp"); } +/// H.264 bitrate control over one second of the demo: the adaptive Match Source preset targets +/// width × height × fps × bpp, the 1080p delivery presets their fixed targets; ffprobe's measured +/// video bitrate stays within the VBR maximum and close to the target. 0.5 to 1.5 s holds the +/// second half of the one-second fade in from black, where every frame changes and each preset +/// spends its whole budget; the shot after it is simple enough that both 1080p presets land at the +/// same bitrate, far under their targets. #[test] fn h264_presets_hit_their_bitrates() { let mut s = demo(); @@ -366,7 +371,7 @@ fn h264_presets_hit_their_bitrates() { let path = dir.path(&format!("{}.mp4", filmcraft_export::presets::preset_key(preset))); s.execute( "file.exportMedia", - json!({"preset": preset, "path": path, "range": "custom", "startSeconds": 0.0, "endSeconds": 3.0, "audio": false, "wait": true}), + json!({"preset": preset, "path": path, "range": "custom", "startSeconds": 0.5, "endSeconds": 1.5, "audio": false, "wait": true}), ) .unwrap(); let Some(j) = probe(&path) else { return }; diff --git a/crates/export/tests/it/main.rs b/crates/export/tests/it/main.rs new file mode 100644 index 000000000..3452f185c --- /dev/null +++ b/crates/export/tests/it/main.rs @@ -0,0 +1,2 @@ +mod pool_trim; +mod renderer_reentrancy; diff --git a/crates/export/tests/pool_trim.rs b/crates/export/tests/it/pool_trim.rs similarity index 100% rename from crates/export/tests/pool_trim.rs rename to crates/export/tests/it/pool_trim.rs diff --git a/crates/export/tests/renderer_reentrancy.rs b/crates/export/tests/it/renderer_reentrancy.rs similarity index 98% rename from crates/export/tests/renderer_reentrancy.rs rename to crates/export/tests/it/renderer_reentrancy.rs index fc6c235bc..2c3a8bd8a 100644 --- a/crates/export/tests/renderer_reentrancy.rs +++ b/crates/export/tests/it/renderer_reentrancy.rs @@ -143,7 +143,7 @@ fn nested_rayon_renderers_finish_every_export_path() { } for case in ["png", "gif", "stepped", "overlap-audio"] { let mut child = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "nested_rayon_renderers_finish_every_export_path", "--nocapture"]) + .args(["--exact", "renderer_reentrancy::nested_rayon_renderers_finish_every_export_path", "--nocapture"]) .env(CHILD, case) .spawn() .unwrap(); diff --git a/crates/h264/README.md b/crates/h264/README.md index 21e7a10f1..634a2977f 100644 --- a/crates/h264/README.md +++ b/crates/h264/README.md @@ -108,7 +108,7 @@ with ffmpeg + libx264; tests print a message and skip when `/opt/homebrew/bin/ff explicit bi-prediction. The expected output is known exactly and is also checked against ffmpeg. - Robustness: avcC input with pts round trip; randomly corrupted, truncated and garbage input never panics or deadlocks (single- and multi-threaded). -- Conformance (`tests/conformance.rs`): every fixture is decoded single-threaded, with 3 threads +- Conformance (`tests/it/conformance.rs`): every fixture is decoded single-threaded, with 3 threads and with every core, and compared byte-for-byte with `ffmpeg -f rawvideo -pix_fmt yuv420p`; on mismatch the first differing frame, plane, sample position and macroblock are reported. Output pts/POC order is checked too, and that no picture is flagged draft without draft mode. Draft @@ -164,12 +164,12 @@ encoder is unavailable. | vt_high | testsrc2+noise 640x360 | VideoToolbox, High | | vt_high_1080p | testsrc2 1920x1080 | VideoToolbox, High 8 Mbit/s | -`cargo test --release -p filmcraft-h264 --test conformance coverage_report -- --ignored --nocapture` +`cargo test --release -p filmcraft-h264 --test it conformance::coverage_report -- --ignored --nocapture` prints which coding tools each fixture exercises. ## Performance -`cargo test --release -p filmcraft-h264 --test perf -- --ignored --nocapture` (1080p High profile, +`cargo test --release -p filmcraft-h264 --test it perf:: -- --ignored --nocapture` (1080p High profile, CABAC, B-pyramid, 120 frames, 8.8 Mbit/s; bit-exactness is verified first). Measured on an Apple M4 Pro (14 cores) while the machine was heavily shared (load average 40-180), so these are lower bounds: diff --git a/crates/h264/tests/fuzz_hostile.rs b/crates/h264/tests/fuzz_hostile.rs index d45af4148..92f7bdc6c 100644 --- a/crates/h264/tests/fuzz_hostile.rs +++ b/crates/h264/tests/fuzz_hostile.rs @@ -1,6 +1,7 @@ //! Hostile-input robustness: header-region sweeps (SPS/PPS/first slice headers), truncation and //! random mutation of small libx264 streams must never panic or run away. Skips without ffmpeg. +#[path = "it/common/mod.rs"] mod common; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -82,6 +83,7 @@ fn run_all(d: &[u8], label: &str) { /// Seconds the whole sweep may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). const DEADLINE_SECS: u64 = 600; +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=400` runs all of it (see [`Depth`]). #[test] fn hostile_streams_never_panic() { filmcraft_testkit::with_deadline(DEADLINE_SECS, sweep); @@ -89,6 +91,7 @@ fn hostile_streams_never_panic() { fn sweep() { let Some(ff) = filmcraft_testkit::ffmpeg_or_skip("h264 fuzz") else { return }; + let depth = Depth::from_env(); for &(name, args, pix) in SEEDS { if !filmcraft_testkit::ffmpeg_has_encoder(&ff, "libx264") { filmcraft_testkit::skip("h264 fuzz", &format!("this ffmpeg has no libx264 encoder (seed {name})")); @@ -97,25 +100,31 @@ fn sweep() { let Some(seed) = make(&ff, name, args, pix) else { panic!("ffmpeg has libx264 but could not encode the {name} seed"); }; - let stride = (seed.len() / 300).max(1); - for n in (0..seed.len()).step_by(stride) { + for n in depth.truncations(seed.len()) { run_all(&seed[..n], &format!("{name} truncate {n}")); } // SPS/PPS/first slice header region: every byte to a few values, every bit for p in 0..seed.len().min(120) { - for v in [0u8, 0xff, 0x80, 0x01, 0x7f] { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(p, i) { + continue; + } let mut d = seed.clone(); d[p] = v; run_all(&d, &format!("{name} byte {p}={v:#x}")); } for b in 0..8 { + if !depth.bit(p, b) { + continue; + } let mut d = seed.clone(); d[p] ^= 1 << b; run_all(&d, &format!("{name} flip {p}.{b}")); } } let mut rng = Rng(0x9e3779b97f4a7c15 ^ name.len() as u64); - for i in 0..iters() { + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { let mut d = seed.clone(); for _ in 0..1 + rng.below(6) { let p = rng.below(d.len()); @@ -125,12 +134,61 @@ fn sweep() { d[p] = rng.next() as u8; } } - run_all(&d, &format!("{name} random {i}")); + if depth.round(i) { + run_all(&d, &format!("{name} random {i}")); + } } } } -/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt). -fn iters() -> usize { - std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()).unwrap_or(400) +const VALUES: [u8; 5] = [0, 0xff, 0x80, 0x01, 0x7f]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 300 truncation +/// points spread over each seed, every header byte, overwrite value and bit flip) plus N random +/// rounds per seed: 400 is the full depth, more hunts longer. Unset, CI runs a third of the +/// truncation points and of the full depth's random rounds and, for each header byte, one value +/// and two bits. A rotation offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC +/// day number), so every case of the full sweep runs within a few days, and a random round's +/// mutations match the same round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 300, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=400 runs everything)"); + Depth { full: false, truncations: 300, rounds: 400, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, p: usize, i: usize) -> bool { + self.full || i == (p + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, p: usize, b: usize) -> bool { + self.full || (b + p + self.rotation % 4).is_multiple_of(4) + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) } diff --git a/crates/h264/tests/common/mod.rs b/crates/h264/tests/it/common/mod.rs similarity index 99% rename from crates/h264/tests/common/mod.rs rename to crates/h264/tests/it/common/mod.rs index e100c3289..2de4d62d2 100644 --- a/crates/h264/tests/common/mod.rs +++ b/crates/h264/tests/it/common/mod.rs @@ -64,7 +64,7 @@ pub const FIXTURES: &[Fixture] = &[ filter: "", args: &["-c:v", "h264_videotoolbox", "-profile:v", "high", "-b:v", "8M"], }, - // performance reference stream (see tests/perf.rs) + // performance reference stream (see tests/it/perf.rs) Fixture { name: "bench_1080p", source: "testsrc2", diff --git a/crates/h264/tests/conformance.rs b/crates/h264/tests/it/conformance.rs similarity index 99% rename from crates/h264/tests/conformance.rs rename to crates/h264/tests/it/conformance.rs index c06694b79..a55d9fe40 100644 --- a/crates/h264/tests/conformance.rs +++ b/crates/h264/tests/it/conformance.rs @@ -1,6 +1,6 @@ //! Bit-exactness against ffmpeg's decoder on libx264-generated fixtures (skipped when ffmpeg is absent). -mod common; +use crate::common; macro_rules! fixture_tests { ($($name:ident),* $(,)?) => { diff --git a/crates/h264/tests/headers.rs b/crates/h264/tests/it/headers.rs similarity index 99% rename from crates/h264/tests/headers.rs rename to crates/h264/tests/it/headers.rs index f0f7a59e3..f27a7ed01 100644 --- a/crates/h264/tests/headers.rs +++ b/crates/h264/tests/it/headers.rs @@ -1,6 +1,6 @@ //! Parameter set / slice header / POC parsing over the fixture streams. -mod common; +use crate::common; use filmcraft_bitstream::{annexb_nals, unescape_rbsp}; use filmcraft_h264::params::{Pps, Sps}; diff --git a/crates/h264/tests/it/main.rs b/crates/h264/tests/it/main.rs new file mode 100644 index 000000000..ceefd95be --- /dev/null +++ b/crates/h264/tests/it/main.rs @@ -0,0 +1,6 @@ +mod common; +mod conformance; +mod headers; +mod perf; +mod pps_validation; +mod robustness; diff --git a/crates/h264/tests/perf.rs b/crates/h264/tests/it/perf.rs similarity index 91% rename from crates/h264/tests/perf.rs rename to crates/h264/tests/it/perf.rs index c8ed4beb8..da744c24b 100644 --- a/crates/h264/tests/perf.rs +++ b/crates/h264/tests/it/perf.rs @@ -1,7 +1,7 @@ //! Decode speed on a 1080p High-profile stream (ignored by default; run with -//! `cargo test --release -p filmcraft-h264 --test perf -- --ignored --nocapture`). +//! `cargo test --release -p filmcraft-h264 --test it perf:: -- --ignored --nocapture`). -mod common; +use crate::common; use filmcraft_h264::Decoder; use std::time::Instant; diff --git a/crates/h264/tests/pps_validation.rs b/crates/h264/tests/it/pps_validation.rs similarity index 100% rename from crates/h264/tests/pps_validation.rs rename to crates/h264/tests/it/pps_validation.rs diff --git a/crates/h264/tests/robustness.rs b/crates/h264/tests/it/robustness.rs similarity index 99% rename from crates/h264/tests/robustness.rs rename to crates/h264/tests/it/robustness.rs index 902ab191d..898d50bef 100644 --- a/crates/h264/tests/robustness.rs +++ b/crates/h264/tests/it/robustness.rs @@ -1,6 +1,6 @@ //! avcC / length-prefixed input and resilience against damaged streams. -mod common; +use crate::common; use filmcraft_bitstream::annexb_nals; use filmcraft_h264::Decoder; diff --git a/crates/h264enc/examples/h264enc_synth.rs b/crates/h264enc/examples/h264enc_synth.rs index 15ab55fd9..3d1b25f4c 100644 --- a/crates/h264enc/examples/h264enc_synth.rs +++ b/crates/h264enc/examples/h264enc_synth.rs @@ -2,7 +2,7 @@ //! //! `cargo run --release -p filmcraft-h264enc --example h264enc_synth -- W H FRAMES [profile] [preset] [bframes] [rate] [slices] [src]` //! profile: baseline|main|high, preset: speed|balanced|quality, rate: qpN | crfN | cbrN | vbrN -#[path = "../tests/common/mod.rs"] +#[path = "../tests/it/common/mod.rs"] mod common; use filmcraft_h264enc::*; diff --git a/crates/h264enc/tests/common/mod.rs b/crates/h264enc/tests/it/common/mod.rs similarity index 100% rename from crates/h264enc/tests/common/mod.rs rename to crates/h264enc/tests/it/common/mod.rs diff --git a/crates/h264enc/tests/crf_level_and_note.rs b/crates/h264enc/tests/it/crf_level_and_note.rs similarity index 98% rename from crates/h264enc/tests/crf_level_and_note.rs rename to crates/h264enc/tests/it/crf_level_and_note.rs index b93ebf66b..58306c073 100644 --- a/crates/h264enc/tests/crf_level_and_note.rs +++ b/crates/h264enc/tests/it/crf_level_and_note.rs @@ -1,7 +1,7 @@ //! Constant quality has no bitrate cap, so the level must allow what low factors spend; and every //! stream says which settings made it. -mod common; +use crate::common; use common::synth; use filmcraft_h264enc::*; diff --git a/crates/h264enc/tests/ffmpeg_oracle.rs b/crates/h264enc/tests/it/ffmpeg_oracle.rs similarity index 99% rename from crates/h264enc/tests/ffmpeg_oracle.rs rename to crates/h264enc/tests/it/ffmpeg_oracle.rs index e19cfc49b..409bf4102 100644 --- a/crates/h264enc/tests/ffmpeg_oracle.rs +++ b/crates/h264enc/tests/it/ffmpeg_oracle.rs @@ -3,7 +3,7 @@ //! Every stream must decode with zero errors, with the exact frame count, and the decoder's output must be //! bit-identical to the encoder's own reconstruction (catches prediction/transform/deblocking mismatches). -mod common; +use crate::common; use common::{Yuv, annexb_nals, ffmpeg, ffmpeg_decode, ffmpeg_source, ffprobe, out_dir, psnr, synth}; use filmcraft_h264enc::*; diff --git a/crates/h264enc/tests/it/main.rs b/crates/h264enc/tests/it/main.rs new file mode 100644 index 000000000..f673a96da --- /dev/null +++ b/crates/h264enc/tests/it/main.rs @@ -0,0 +1,3 @@ +mod common; +mod crf_level_and_note; +mod ffmpeg_oracle; diff --git a/crates/hevc/README.md b/crates/hevc/README.md index 43c266eda..59c762aed 100644 --- a/crates/hevc/README.md +++ b/crates/hevc/README.md @@ -96,7 +96,7 @@ with ffmpeg; tests print a message and skip when ffmpeg is absent). ffmpeg. - Robustness: hvcC input with pts round trip; randomly corrupted, truncated and garbage input never panics or deadlocks (single- and multi-threaded). -- Conformance (`tests/conformance.rs`): every fixture is decoded with 1 thread, 3 threads and every +- Conformance (`tests/it/conformance.rs`): every fixture is decoded with 1 thread, 3 threads and every core and compared sample-exactly with `ffmpeg -f rawvideo -pix_fmt yuv420p / yuv420p10le`; on mismatch the first differing frame, plane, sample, CTB and 8x8 block are reported. Output pts/POC order is checked too, and that no picture is flagged draft without draft mode. `compare_detects_mismatch` @@ -135,12 +135,12 @@ libx265 unless noted (x265 enables WPP by default, so most streams carry entry p | vt_main / vt_main10 | 640x360 | Apple VideoToolbox (hevc_videotoolbox), macOS only | | synthetic (7 streams) | 96x64 | tiles, PCM, slices, dependent segments, LTR, list modification | -`cargo test -p filmcraft-hevc --test conformance coverage_report -- --ignored --nocapture` prints the +`cargo test -p filmcraft-hevc --test it conformance::coverage_report -- --ignored --nocapture` prints the decoder statistics per fixture. ## Performance -`cargo test --release -p filmcraft-hevc --test perf -- --ignored --nocapture` (bit-exactness is +`cargo test --release -p filmcraft-hevc --test it perf:: -- --ignored --nocapture` (bit-exactness is verified first). Apple M4 Pro (14 cores), measured while the machine was heavily shared (load average 50-100), so these are lower bounds: diff --git a/crates/hevc/tests/fuzz_hostile.rs b/crates/hevc/tests/fuzz_hostile.rs index ac6829315..c74400674 100644 --- a/crates/hevc/tests/fuzz_hostile.rs +++ b/crates/hevc/tests/fuzz_hostile.rs @@ -1,6 +1,7 @@ //! Hostile-input robustness: header-region sweeps (SPS/PPS/first slice headers), truncation and //! random mutation of small libx265 streams must never panic or run away. Skips without ffmpeg. +#[path = "it/common/mod.rs"] mod common; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -83,6 +84,7 @@ fn run_all(d: &[u8], label: &str) { /// Seconds the whole sweep may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). const DEADLINE_SECS: u64 = 600; +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=400` runs all of it (see [`Depth`]). #[test] fn hostile_streams_never_panic() { filmcraft_testkit::with_deadline(DEADLINE_SECS, sweep); @@ -90,6 +92,7 @@ fn hostile_streams_never_panic() { fn sweep() { let Some(ff) = filmcraft_testkit::ffmpeg_or_skip("hevc fuzz") else { return }; + let depth = Depth::from_env(); for &(name, enc, args, pix) in SEEDS { if !filmcraft_testkit::ffmpeg_has_encoder(&ff, enc) { filmcraft_testkit::skip("hevc fuzz", &format!("this ffmpeg has no {enc} encoder (seed {name})")); @@ -98,25 +101,31 @@ fn sweep() { let Some(seed) = make(&ff, name, enc, args, pix) else { panic!("ffmpeg has {enc} but could not encode the {name} seed"); }; - let stride = (seed.len() / 300).max(1); - for n in (0..seed.len()).step_by(stride) { + for n in depth.truncations(seed.len()) { run_all(&seed[..n], &format!("{name} truncate {n}")); } // SPS/PPS/first slice header region: every byte to a few values, every bit for p in 0..seed.len().min(120) { - for v in [0u8, 0xff, 0x80, 0x01, 0x7f] { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(p, i) { + continue; + } let mut d = seed.clone(); d[p] = v; run_all(&d, &format!("{name} byte {p}={v:#x}")); } for b in 0..8 { + if !depth.bit(p, b) { + continue; + } let mut d = seed.clone(); d[p] ^= 1 << b; run_all(&d, &format!("{name} flip {p}.{b}")); } } let mut rng = Rng(0x9e3779b97f4a7c15 ^ name.len() as u64); - for i in 0..iters() { + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { let mut d = seed.clone(); for _ in 0..1 + rng.below(6) { let p = rng.below(d.len()); @@ -126,12 +135,61 @@ fn sweep() { d[p] = rng.next() as u8; } } - run_all(&d, &format!("{name} random {i}")); + if depth.round(i) { + run_all(&d, &format!("{name} random {i}")); + } } } } -/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt). -fn iters() -> usize { - std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()).unwrap_or(400) +const VALUES: [u8; 5] = [0, 0xff, 0x80, 0x01, 0x7f]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 300 truncation +/// points spread over each seed, every header byte, overwrite value and bit flip) plus N random +/// rounds per seed: 400 is the full depth, more hunts longer. Unset, CI runs a third of the +/// truncation points and of the full depth's random rounds and, for each header byte, one value +/// and two bits. A rotation offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC +/// day number), so every case of the full sweep runs within a few days, and a random round's +/// mutations match the same round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 300, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=400 runs everything)"); + Depth { full: false, truncations: 300, rounds: 400, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, p: usize, i: usize) -> bool { + self.full || i == (p + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, p: usize, b: usize) -> bool { + self.full || (b + p + self.rotation % 4).is_multiple_of(4) + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) } diff --git a/crates/hevc/tests/common/mod.rs b/crates/hevc/tests/it/common/mod.rs similarity index 99% rename from crates/hevc/tests/common/mod.rs rename to crates/hevc/tests/it/common/mod.rs index a8dcb8da2..61c1033f9 100644 --- a/crates/hevc/tests/common/mod.rs +++ b/crates/hevc/tests/it/common/mod.rs @@ -73,7 +73,7 @@ pub const FIXTURES: &[Fixture] = &[ fx!("odd_1918x1078", "testsrc2", 1918, 1078, 3, NOISE, 8, ["-preset", "veryfast", "-x265-params", "log-level=error"]), fx!("hd_1080p", "testsrc2", 1920, 1080, 3, "", 8, ["-preset", "fast", "-x265-params", "log-level=error"]), fx!("uhd_4k_main10", "testsrc2", 3840, 2160, 2, "", 10, ["-preset", "ultrafast", "-x265-params", "log-level=error"]), - // performance reference streams (tests/perf.rs) + // performance reference streams (tests/it/perf.rs) fx!("bench_1080p", "testsrc2", 1920, 1080, 60, "noise=alls=3:allf=t", 8, ["-preset", "medium", "-crf", "24", "-x265-params", "log-level=error"]), fx!("bench_4k", "testsrc2", 3840, 2160, 20, "noise=alls=3:allf=t", 10, ["-preset", "fast", "-crf", "26", "-x265-params", "log-level=error"]), // Apple VideoToolbox (hardware encoder, macOS only) diff --git a/crates/hevc/tests/conformance.rs b/crates/hevc/tests/it/conformance.rs similarity index 99% rename from crates/hevc/tests/conformance.rs rename to crates/hevc/tests/it/conformance.rs index 4fa8b8cbe..7b50cbaf0 100644 --- a/crates/hevc/tests/conformance.rs +++ b/crates/hevc/tests/it/conformance.rs @@ -1,7 +1,7 @@ //! Bit-exactness against ffmpeg's decoder on libx265 / VideoToolbox fixtures (skipped when ffmpeg is //! absent). -mod common; +use crate::common; macro_rules! fixture_tests { ($($name:ident),* $(,)?) => { diff --git a/crates/hevc/tests/it/main.rs b/crates/hevc/tests/it/main.rs new file mode 100644 index 000000000..be033f189 --- /dev/null +++ b/crates/hevc/tests/it/main.rs @@ -0,0 +1,4 @@ +mod common; +mod conformance; +mod perf; +mod robustness; diff --git a/crates/hevc/tests/perf.rs b/crates/hevc/tests/it/perf.rs similarity index 91% rename from crates/hevc/tests/perf.rs rename to crates/hevc/tests/it/perf.rs index 4091166b6..ab8144a12 100644 --- a/crates/hevc/tests/perf.rs +++ b/crates/hevc/tests/it/perf.rs @@ -1,7 +1,7 @@ //! Decode speed (ignored by default; run with -//! `cargo test --release -p filmcraft-hevc --test perf -- --ignored --nocapture`). +//! `cargo test --release -p filmcraft-hevc --test it perf:: -- --ignored --nocapture`). -mod common; +use crate::common; use filmcraft_hevc::Decoder; use std::time::Instant; diff --git a/crates/hevc/tests/robustness.rs b/crates/hevc/tests/it/robustness.rs similarity index 99% rename from crates/hevc/tests/robustness.rs rename to crates/hevc/tests/it/robustness.rs index 075d5fde2..50f35b0d4 100644 --- a/crates/hevc/tests/robustness.rs +++ b/crates/hevc/tests/it/robustness.rs @@ -1,6 +1,6 @@ //! hvcC / length-prefixed input and resilience against damaged streams. -mod common; +use crate::common; use filmcraft_bitstream::annexb_nals; use filmcraft_hevc::Decoder; diff --git a/crates/interchange/README.md b/crates/interchange/README.md index edaff851f..b0cd33577 100644 --- a/crates/interchange/README.md +++ b/crates/interchange/README.md @@ -25,7 +25,7 @@ read off files it exports, never from the application itself. The other formats' editions are listed in the module docs (`edl`, `fcp7`, `fcpxml`, `otio`, `ale`). **Invariant — Unicode media URLs:** file URL prefix probes must not panic when their byte boundary -falls inside a UTF-8 character. Timeline importers use the shared converter; `tests/file_urls.rs` +falls inside a UTF-8 character. Timeline importers use the shared converter; `tests/it/file_urls.rs` checks expected Unicode paths through the public converter, resolver and EDL importer. ## AAF and OMF @@ -85,7 +85,7 @@ OMF carries no video at all. ## Tests -- `tests/aaf_omf.rs`: AAF round trips at 25, 29.97 DF, 23.976 and 59.94 DF (clips, gaps, +- `tests/it/aaf_omf.rs`: AAF round trips at 25, 29.97 DF, 23.976 and 59.94 DF (clips, gaps, two-sided and one-sided transitions, links, markers, start timecodes, constant and keyframed gain, media markers and timecode); 512-byte sectors; breakout to mono; embedded trimmed audio (sample data and source offsets); separate consolidated files with a video mixdown; OMF round @@ -93,7 +93,7 @@ OMF carries no video at all. random corruption never panic. Nested sequences: AAF round trips as compositions (nests in nests, a nest used twice, another frame and sample rate, breakout to mono), OMF with the rendered sound, the audio needs of nests, a sequence inside itself. -- `tests/uncarried_clips.rs`: every format's export report names each graphic clip and adjustment +- `tests/it/uncarried_clips.rs`: every format's export report names each graphic clip and adjustment layer once, as a warning; a sequence without them reports nothing; what a re-import gives back matches the report; the FCP7 XML import names a clip item whose file is not defined. - `src/aaf/tests.rs`: written files checked against the required properties of every class diff --git a/crates/interchange/tests/aaf_omf.rs b/crates/interchange/tests/it/aaf_omf.rs similarity index 99% rename from crates/interchange/tests/aaf_omf.rs rename to crates/interchange/tests/it/aaf_omf.rs index 3e16492f2..4a065677c 100644 --- a/crates/interchange/tests/aaf_omf.rs +++ b/crates/interchange/tests/it/aaf_omf.rs @@ -1,6 +1,6 @@ //! AAF and OMF: export → import round trips (no external oracle exists for these formats). -mod support; +use crate::support; use filmcraft_interchange::aaf::{self, AafOptions}; use filmcraft_interchange::essence::{AudioEssence, EssenceData, EssenceKey, MediaOptions, MixdownVideo, NeedOptions, NestNeeds, audio_needs}; diff --git a/crates/interchange/tests/api.rs b/crates/interchange/tests/it/api.rs similarity index 99% rename from crates/interchange/tests/api.rs rename to crates/interchange/tests/it/api.rs index 6a212793c..7d19eb3a9 100644 --- a/crates/interchange/tests/api.rs +++ b/crates/interchange/tests/it/api.rs @@ -1,6 +1,6 @@ //! Crate API: detection, merging fragments into a project, timecode rebasing. -mod support; +use crate::support; use std::collections::HashSet; diff --git a/crates/interchange/tests/edl.rs b/crates/interchange/tests/it/edl.rs similarity index 99% rename from crates/interchange/tests/edl.rs rename to crates/interchange/tests/it/edl.rs index 14100d146..db7369589 100644 --- a/crates/interchange/tests/edl.rs +++ b/crates/interchange/tests/it/edl.rs @@ -1,6 +1,6 @@ //! CMX 3600 EDL: golden documents (hand-written from the public CMX 3600 conventions) and round trips. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, ImportOptions, ReelMode, detect, export, export_edl_per_track, import, import_with}; use filmcraft_project::{ItemKind, MediaRef, TrackKind, TransitionAlign}; diff --git a/crates/interchange/tests/fcp7.rs b/crates/interchange/tests/it/fcp7.rs similarity index 99% rename from crates/interchange/tests/fcp7.rs rename to crates/interchange/tests/it/fcp7.rs index aad1d5f0e..663ce7337 100644 --- a/crates/interchange/tests/fcp7.rs +++ b/crates/interchange/tests/it/fcp7.rs @@ -1,7 +1,7 @@ //! Final Cut Pro 7 XML (xmeml): a hand-written Premiere-style document (written from the public //! xmeml element descriptions) and export → import round trips. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, Imported, detect, export, import}; use filmcraft_media::Generator; diff --git a/crates/interchange/tests/fcpxml.rs b/crates/interchange/tests/it/fcpxml.rs similarity index 99% rename from crates/interchange/tests/fcpxml.rs rename to crates/interchange/tests/it/fcpxml.rs index 740852a7a..2bf6f3c46 100644 --- a/crates/interchange/tests/fcpxml.rs +++ b/crates/interchange/tests/it/fcpxml.rs @@ -1,7 +1,7 @@ //! FCPXML: a hand-written document (from the public FCPXML element reference) with connected //! clips, a transition, a retimed clip, a gap and a compound clip; plus export → import round trips. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, FcpxmlVersion, Format, detect, import}; use filmcraft_project::{ItemKind, Keyframe, Label, Marker, MarkerId, MarkerKind, MediaRef, Param, ParamValue, Project, TrackKind, TransitionAlign}; diff --git a/crates/interchange/tests/file_urls.rs b/crates/interchange/tests/it/file_urls.rs similarity index 100% rename from crates/interchange/tests/file_urls.rs rename to crates/interchange/tests/it/file_urls.rs diff --git a/crates/interchange/tests/it/main.rs b/crates/interchange/tests/it/main.rs new file mode 100644 index 000000000..81da996e9 --- /dev/null +++ b/crates/interchange/tests/it/main.rs @@ -0,0 +1,12 @@ +mod aaf_omf; +mod api; +mod edl; +mod fcp7; +mod fcpxml; +mod file_urls; +mod media_marker_export; +mod nested_sequence_markers; +mod otio; +mod point_markers; +mod support; +mod uncarried_clips; diff --git a/crates/interchange/tests/media_marker_export.rs b/crates/interchange/tests/it/media_marker_export.rs similarity index 99% rename from crates/interchange/tests/media_marker_export.rs rename to crates/interchange/tests/it/media_marker_export.rs index 752cc8b5d..e209b0693 100644 --- a/crates/interchange/tests/media_marker_export.rs +++ b/crates/interchange/tests/it/media_marker_export.rs @@ -1,6 +1,6 @@ //! Source-media (master clip) markers must survive FCP7 XML export for xmeml 4 and 5. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, export, import}; use filmcraft_project::{Label, Marker, MarkerId, MarkerKind, TrackKind}; diff --git a/crates/interchange/tests/nested_sequence_markers.rs b/crates/interchange/tests/it/nested_sequence_markers.rs similarity index 99% rename from crates/interchange/tests/nested_sequence_markers.rs rename to crates/interchange/tests/it/nested_sequence_markers.rs index 9f69b755e..6e4eca92b 100644 --- a/crates/interchange/tests/nested_sequence_markers.rs +++ b/crates/interchange/tests/it/nested_sequence_markers.rs @@ -4,7 +4,7 @@ //! sequence's markers when the using clip has none, and must not put clip instance markers on the //! nested sequence. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format}; use filmcraft_project::{Label, Marker, MarkerId, MarkerKind, TrackKind}; diff --git a/crates/interchange/tests/otio.rs b/crates/interchange/tests/it/otio.rs similarity index 99% rename from crates/interchange/tests/otio.rs rename to crates/interchange/tests/it/otio.rs index 0b952add4..857de1e79 100644 --- a/crates/interchange/tests/otio.rs +++ b/crates/interchange/tests/it/otio.rs @@ -1,7 +1,7 @@ //! OpenTimelineIO: a hand-written plain OTIO document (from the public OTIO schema docs) with gaps, //! transitions, markers and legacy `Clip.1`; lossless round trips through `metadata.filmcraft`. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, detect, import}; use filmcraft_media::Generator; diff --git a/crates/interchange/tests/point_markers.rs b/crates/interchange/tests/it/point_markers.rs similarity index 99% rename from crates/interchange/tests/point_markers.rs rename to crates/interchange/tests/it/point_markers.rs index e0ffb5616..ba96cec58 100644 --- a/crates/interchange/tests/point_markers.rs +++ b/crates/interchange/tests/it/point_markers.rs @@ -1,6 +1,6 @@ //! FCPXML point markers: a one-frame `` is a FilmCraft point (duration 0), not a range. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, import}; use filmcraft_project::{Label, Marker, MarkerId, MarkerKind, Project, TrackKind}; diff --git a/crates/interchange/tests/support/mod.rs b/crates/interchange/tests/it/support/mod.rs similarity index 100% rename from crates/interchange/tests/support/mod.rs rename to crates/interchange/tests/it/support/mod.rs diff --git a/crates/interchange/tests/uncarried_clips.rs b/crates/interchange/tests/it/uncarried_clips.rs similarity index 99% rename from crates/interchange/tests/uncarried_clips.rs rename to crates/interchange/tests/it/uncarried_clips.rs index 2f0902e26..136802897 100644 --- a/crates/interchange/tests/uncarried_clips.rs +++ b/crates/interchange/tests/it/uncarried_clips.rs @@ -2,7 +2,7 @@ //! export used to lose them without a word (FCP7 XML, OTIO, EDL) or with a line that named //! nothing (FCPXML, AAF): every such clip is now named in the export report. -mod support; +use crate::support; use filmcraft_interchange::{ExportOptions, Format, Report, export, import}; use filmcraft_project::{ItemId, ItemKind, Label, ParamValue, Project, TrackKind, find_effect}; diff --git a/crates/isobmff/README.md b/crates/isobmff/README.md index 47c31bb43..24cea2e43 100644 --- a/crates/isobmff/README.md +++ b/crates/isobmff/README.md @@ -118,18 +118,18 @@ let bytes = w.finish_faststart()?.into_inner(); // or finish() for moov-at-end ## Tests - Unit tests cover the box reader and hand-built `stbl` edge cases (stz2, co64, ctts v1, empty edits, 64-bit and size-0 boxes, uuid/wide/skip), codec config round-trips, and allocation guards. -- Property tests (`tests/roundtrip.rs`) write random multi-track files, both progressive (MP4/MOV, +- Property tests (`tests/it/roundtrip.rs`) write random multi-track files, both progressive (MP4/MOV, faststart, PCM, edits) and fragmented, then read them back and compare sample tables and bytes. -- ffprobe oracle tests (`tests/oracle_demux.rs`) run on ffmpeg-generated fixtures in +- ffprobe oracle tests (`tests/it/oracle_demux.rs`) run on ffmpeg-generated fixtures in `target/fixtures/isobmff/`: H.264 with B-frames (MP4 and MOV), HEVC, ProRes 422/4444, MJPEG, DNxHR, PCM s16le/s16be/s24le/f32le, AAC, ALAC, FLAC, Opus, AC-3, VP9, AV1, fragmented (with and without default-base-is-moof), faststart, an edit-list cut, `tmcd` at 29.97 DF and 23.976, and colr/pasp. They compare packet offsets, sizes, pts/dts, durations and key flags, plus stream parameters. Tests skip with a message when ffmpeg or an encoder is missing. -- Muxer oracle tests (`tests/oracle_mux.rs`) remux fixtures with our writers. They check that +- Muxer oracle tests (`tests/it/oracle_mux.rs`) remux fixtures with our writers. They check that `ffmpeg -v error -f null` reports nothing, that ffprobe packets and stream parameters match the source, and that timecode, the AAC priming edit, ipcm and fMP4 output are correct. -- Mutation fuzzing (`tests/fuzz.rs`) uses a deterministic seed to mutate and truncate synthetic and +- Mutation fuzzing (`tests/it/fuzz.rs`) uses a deterministic seed to mutate and truncate synthetic and ffmpeg files, and checks that nothing panics. ## Limitations / not yet supported diff --git a/crates/isobmff/tests/common/mod.rs b/crates/isobmff/tests/it/common/mod.rs similarity index 100% rename from crates/isobmff/tests/common/mod.rs rename to crates/isobmff/tests/it/common/mod.rs diff --git a/crates/isobmff/tests/fuzz.rs b/crates/isobmff/tests/it/fuzz.rs similarity index 99% rename from crates/isobmff/tests/fuzz.rs rename to crates/isobmff/tests/it/fuzz.rs index b6d2fcea6..17c376a5e 100644 --- a/crates/isobmff/tests/fuzz.rs +++ b/crates/isobmff/tests/it/fuzz.rs @@ -1,6 +1,6 @@ //! Deterministic mutation fuzzing: the demuxer must never panic on corrupted input. -mod common; +use crate::common; use filmcraft_isobmff::*; use std::io::Cursor; diff --git a/crates/isobmff/tests/it/main.rs b/crates/isobmff/tests/it/main.rs new file mode 100644 index 000000000..48e623e19 --- /dev/null +++ b/crates/isobmff/tests/it/main.rs @@ -0,0 +1,5 @@ +mod common; +mod fuzz; +mod oracle_demux; +mod oracle_mux; +mod roundtrip; diff --git a/crates/isobmff/tests/oracle_demux.rs b/crates/isobmff/tests/it/oracle_demux.rs similarity index 99% rename from crates/isobmff/tests/oracle_demux.rs rename to crates/isobmff/tests/it/oracle_demux.rs index 45ffe3b58..9e2cd0b28 100644 --- a/crates/isobmff/tests/oracle_demux.rs +++ b/crates/isobmff/tests/it/oracle_demux.rs @@ -1,6 +1,6 @@ //! Oracle tests: compare our demuxer's sample tables and codec parameters with ffprobe. -mod common; +use crate::common; use common::*; use filmcraft_isobmff::{CodecConfig, Mp4File, TrackKind, open}; diff --git a/crates/isobmff/tests/oracle_mux.rs b/crates/isobmff/tests/it/oracle_mux.rs similarity index 99% rename from crates/isobmff/tests/oracle_mux.rs rename to crates/isobmff/tests/it/oracle_mux.rs index 4b9144241..5383211f6 100644 --- a/crates/isobmff/tests/oracle_mux.rs +++ b/crates/isobmff/tests/it/oracle_mux.rs @@ -1,6 +1,6 @@ //! Muxer oracle tests: remux ffmpeg-made files with our writers and validate with ffmpeg/ffprobe. -mod common; +use crate::common; use common::*; use filmcraft_isobmff::*; diff --git a/crates/isobmff/tests/roundtrip.rs b/crates/isobmff/tests/it/roundtrip.rs similarity index 100% rename from crates/isobmff/tests/roundtrip.rs rename to crates/isobmff/tests/it/roundtrip.rs diff --git a/crates/media/tests/exr_oracle.rs b/crates/media/tests/it/exr_oracle.rs similarity index 100% rename from crates/media/tests/exr_oracle.rs rename to crates/media/tests/it/exr_oracle.rs diff --git a/crates/media/tests/hdr_oracle.rs b/crates/media/tests/it/hdr_oracle.rs similarity index 100% rename from crates/media/tests/hdr_oracle.rs rename to crates/media/tests/it/hdr_oracle.rs diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs new file mode 100644 index 000000000..43b1a5bcb --- /dev/null +++ b/crates/media/tests/it/main.rs @@ -0,0 +1,3 @@ +mod exr_oracle; +mod hdr_oracle; +mod tiff_ycbcr; diff --git a/crates/media/tests/tiff_ycbcr.rs b/crates/media/tests/it/tiff_ycbcr.rs similarity index 100% rename from crates/media/tests/tiff_ycbcr.rs rename to crates/media/tests/it/tiff_ycbcr.rs diff --git a/crates/mpeg2v/README.md b/crates/mpeg2v/README.md index 90d6d758d..2ccc86feb 100644 --- a/crates/mpeg2v/README.md +++ b/crates/mpeg2v/README.md @@ -80,7 +80,7 @@ are compared within a tolerance: **per frame max |Δ| ≤ 4 and PSNR ≥ 58 dB, samples differing**. Frame count, display order, picture types, interlacing and field order are exact (ffprobe). -`tests/oracle.rs` (FFmpeg-encoded elementary streams): +`tests/it/oracle.rs` (FFmpeg-encoded elementary streams): | Fixture | Content | Max Δ | Differing | PSNR | |---|---|---|---|---| @@ -113,7 +113,7 @@ FFmpeg's integer IDCT saturates differently from the exact one. ## Performance -`cargo test --release -p filmcraft-mpeg2v --test perf -- --ignored --nocapture` decodes the +`cargo test --release -p filmcraft-mpeg2v --test it perf:: -- --ignored --nocapture` decodes the 1080i fixture (1920×1080 4:2:0, 25 Mb/s, IBBP, field MC/DCT; FFmpeg-encoded) on an M4 Pro (14 cores) that was heavily loaded by other jobs (load average ≈ 190), best of five runs: **271 fps on one thread, 274 fps with slice rows on rayon** (runs under heavier momentary load @@ -129,5 +129,5 @@ fell to 77–150 fps). These are lower bounds. rounding, field addressing, edge clamping. - `src/tests.rs`: a hand-built intra picture decodes to its DC; truncation at every byte and 3000 random mutations never panic. -- `src/synth_tests.rs`, `tests/oracle.rs`: see above (single-threaded decode identical to the +- `src/synth_tests.rs`, `tests/it/oracle.rs`: see above (single-threaded decode identical to the threaded one). diff --git a/crates/mpeg2v/src/synth_tests.rs b/crates/mpeg2v/src/synth_tests.rs index 0d4dd5453..33baf11a8 100644 --- a/crates/mpeg2v/src/synth_tests.rs +++ b/crates/mpeg2v/src/synth_tests.rs @@ -725,7 +725,7 @@ fn check(cfg: Cfg) { } let psnr = 10.0 * (255f64 * 255.0 * n as f64 / sse.max(1e-9)).log10(); println!("{}: {} frames, max diff {worst}, {:.3}% differ, PSNR {psnr:.2} dB", cfg.name, pics.len(), 100.0 * differ as f64 / n as f64); - // the same criteria as the ffmpeg-encoded fixtures (tests/oracle.rs) + // the same criteria as the ffmpeg-encoded fixtures (tests/it/oracle.rs) assert!(worst <= 4 && psnr >= 58.0, "{} (seed {:#x}): max diff {worst}, PSNR {psnr:.2}, first at {first_bad:?}", cfg.name, cfg.seed); } diff --git a/crates/mpeg2v/src/tests.rs b/crates/mpeg2v/src/tests.rs index ca5f2e5e5..eeae62aae 100644 --- a/crates/mpeg2v/src/tests.rs +++ b/crates/mpeg2v/src/tests.rs @@ -1,5 +1,5 @@ //! Unit tests over hand-built streams (headers, robustness). Decoding accuracy against ffmpeg is -//! in `tests/oracle.rs`. +//! in `tests/it/oracle.rs`. use super::*; diff --git a/crates/mpeg2v/tests/fuzz_hostile.rs b/crates/mpeg2v/tests/fuzz_hostile.rs index d293af2d2..5d3aa21a2 100644 --- a/crates/mpeg2v/tests/fuzz_hostile.rs +++ b/crates/mpeg2v/tests/fuzz_hostile.rs @@ -2,6 +2,7 @@ //! sweeps over a small ffmpeg-generated stream must never panic or run away (errors are fine). //! Skips when ffmpeg is missing. +#[path = "it/common/mod.rs"] mod common; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -82,6 +83,7 @@ fn run_all(d: &[u8], label: &str) { /// Seconds the whole sweep may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). const DEADLINE_SECS: u64 = 600; +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=2500` runs all of it (see [`Depth`]). #[test] fn hostile_streams_never_panic() { filmcraft_testkit::with_deadline(DEADLINE_SECS, sweep); @@ -89,6 +91,7 @@ fn hostile_streams_never_panic() { fn sweep() { let Some(ff) = filmcraft_testkit::ffmpeg_or_skip("mpeg2v fuzz") else { return }; + let depth = Depth::from_env(); for spec in &SPECS { if !filmcraft_testkit::ffmpeg_has_encoder(&ff, "mpeg2video") { filmcraft_testkit::skip("mpeg2v fuzz", &format!("this ffmpeg has no mpeg2video encoder (seed {})", spec.name)); @@ -97,24 +100,29 @@ fn sweep() { let Some(path) = common::make(&ff, spec) else { panic!("ffmpeg has mpeg2video but could not encode the {} seed", spec.name); }; - fuzz(spec.name, &std::fs::read(path).unwrap()); + fuzz(&depth, spec.name, &std::fs::read(path).unwrap()); } } -fn fuzz(name: &str, seed: &[u8]) { +fn fuzz(depth: &Depth, name: &str, seed: &[u8]) { // every truncation length (stride keeps it quick on larger seeds) - let stride = (seed.len() / 600).max(1); - for n in (0..seed.len()).step_by(stride) { + for n in depth.truncations(seed.len()) { run_all(&seed[..n], &format!("{name} truncate {n}")); } // header region: every byte to 0 / 0xff / 0x80 / each single bit for p in 0..seed.len().min(96) { - for v in [0u8, 0xff, 0x80, 0x01] { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(p, i) { + continue; + } let mut d = seed.to_vec(); d[p] = v; run_all(&d, &format!("{name} byte {p}={v:#x}")); } for b in 0..8 { + if !depth.bit(p, b) { + continue; + } let mut d = seed.to_vec(); d[p] ^= 1 << b; run_all(&d, &format!("{name} flip {p}.{b}")); @@ -122,7 +130,8 @@ fn fuzz(name: &str, seed: &[u8]) { } // random multi-mutations let mut rng = Rng(0x9e3779b97f4a7c15); - for i in 0..iters() { + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { let mut d = seed.to_vec(); for _ in 0..1 + rng.below(6) { let p = rng.below(d.len()); @@ -132,11 +141,60 @@ fn fuzz(name: &str, seed: &[u8]) { d[p] = rng.next() as u8; } } - run_all(&d, &format!("{name} random {i}")); + if depth.round(i) { + run_all(&d, &format!("{name} random {i}")); + } } } -/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt). -fn iters() -> usize { - std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()).unwrap_or(2500) +const VALUES: [u8; 4] = [0, 0xff, 0x80, 0x01]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 600 truncation +/// points spread over each seed, every header byte, overwrite value and bit flip) plus N random +/// rounds per seed: 2500 is the full depth, more hunts longer. Unset, CI runs a third of the +/// truncation points and of the full depth's random rounds and, for each header byte, one value +/// and two bits. A rotation offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC +/// day number), so every case of the full sweep runs within a few days, and a random round's +/// mutations match the same round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 600, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=2500 runs everything)"); + Depth { full: false, truncations: 600, rounds: 2500, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, p: usize, i: usize) -> bool { + self.full || i == (p + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, p: usize, b: usize) -> bool { + self.full || (b + p + self.rotation % 4).is_multiple_of(4) + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) } diff --git a/crates/mpeg2v/tests/common/mod.rs b/crates/mpeg2v/tests/it/common/mod.rs similarity index 100% rename from crates/mpeg2v/tests/common/mod.rs rename to crates/mpeg2v/tests/it/common/mod.rs diff --git a/crates/mpeg2v/tests/hostile_dims.rs b/crates/mpeg2v/tests/it/hostile_dims.rs similarity index 100% rename from crates/mpeg2v/tests/hostile_dims.rs rename to crates/mpeg2v/tests/it/hostile_dims.rs diff --git a/crates/mpeg2v/tests/it/main.rs b/crates/mpeg2v/tests/it/main.rs new file mode 100644 index 000000000..bf473ddbb --- /dev/null +++ b/crates/mpeg2v/tests/it/main.rs @@ -0,0 +1,4 @@ +mod common; +mod hostile_dims; +mod oracle; +mod perf; diff --git a/crates/mpeg2v/tests/oracle.rs b/crates/mpeg2v/tests/it/oracle.rs similarity index 99% rename from crates/mpeg2v/tests/oracle.rs rename to crates/mpeg2v/tests/it/oracle.rs index eed298c7d..00473c955 100644 --- a/crates/mpeg2v/tests/oracle.rs +++ b/crates/mpeg2v/tests/it/oracle.rs @@ -3,7 +3,7 @@ //! definition (any IEEE 1180-accurate IDCT conforms), so frames are compared within a tolerance; //! frame count and display order are exact. -mod common; +use crate::common; use common::*; diff --git a/crates/mpeg2v/tests/perf.rs b/crates/mpeg2v/tests/it/perf.rs similarity index 87% rename from crates/mpeg2v/tests/perf.rs rename to crates/mpeg2v/tests/it/perf.rs index 44281499c..10aaa1d6d 100644 --- a/crates/mpeg2v/tests/perf.rs +++ b/crates/mpeg2v/tests/it/perf.rs @@ -1,7 +1,7 @@ -//! Decode speed (ignored; run with `cargo test --release -p filmcraft-mpeg2v --test perf -- -//! --ignored --nocapture`). Uses the 1080i fixture of `tests/oracle.rs` (ffmpeg-encoded, 25 Mb/s). +//! Decode speed (ignored; run with `cargo test --release -p filmcraft-mpeg2v --test it perf:: -- +//! --ignored --nocapture`). Uses the 1080i fixture of `tests/it/oracle.rs` (ffmpeg-encoded, 25 Mb/s). -mod common; +use crate::common; use common::*; diff --git a/crates/mpegts/README.md b/crates/mpegts/README.md index 15bb9101e..a5543992c 100644 --- a/crates/mpegts/README.md +++ b/crates/mpegts/README.md @@ -65,7 +65,7 @@ let au: Vec = file.read_unit(&bytes, v, 0)?; // the first access unit 33-bit unwrap; truncation every 37 bytes and 400 random mutations never panic. - `src/es.rs`, `src/pes.rs`: frame sizes (MPEG audio layers I-III, MPEG-2 LSF, AC-3 at all rates), MPEG-2 and MPEG-1 PES headers. -- `crates/codecs/tests/mpeg_oracle.rs`: FFmpeg-written TS / M2TS / VOB / MPEG-1 system files — +- `crates/codecs/tests/it/mpeg_oracle.rs`: FFmpeg-written TS / M2TS / VOB / MPEG-1 system files — frame counts and every frame's PTS equal ffprobe's, decoded media against FFmpeg (see `filmcraft-codecs`). diff --git a/crates/mxf/README.md b/crates/mxf/README.md index 043772477..070045b98 100644 --- a/crates/mxf/README.md +++ b/crates/mxf/README.md @@ -129,7 +129,7 @@ w.finish()?; - `src/tests.rs`: a small OP1a writer builds synthetic files (temporal offsets, key flags, PCM, timecode); open, presentation order, sample bytes, PCM values; truncation at every 37 bytes and 600 random mutations never panic and only list fully-present samples. -- `crates/codecs/tests/mxf_oracle.rs` (FFmpeg-written fixtures, FFmpeg as the decode oracle): +- `crates/codecs/tests/it/mxf_oracle.rs` (FFmpeg-written fixtures, FFmpeg as the decode oracle): H.264 long-GOP with B pictures (bit-exact every frame + 25 random seeks), H.264 29.97 DF timecode, DNxHR LB (±2), ProRes 422 (±1), MPEG-2 long GOP, XDCAM HD422 and D-10 (±4), OP-Atom VC-3 (clip-wrapped) and PCM, D-10 AES3 audio; PCM sample-exact in every file; diff --git a/crates/ogg/README.md b/crates/ogg/README.md index a3d7f9196..851b9863e 100644 --- a/crates/ogg/README.md +++ b/crates/ogg/README.md @@ -37,6 +37,6 @@ let packet = ogg.read_packet(&bytes, s, k)?; ## Tests `src/tests.rs` writes synthetic pages: timing with pre-skip and end trimming, packets across pages, -a corrupted page (CRC), every truncation and 300 mutations. `crates/codecs/tests/ogg_oracle.rs` +a corrupted page (CRC), every truncation and 300 mutations. `crates/codecs/tests/it/ogg_oracle.rs` compares with FFmpeg's libopus / Vorbis decodes (exact length; SNR ≥ 50 dB CELT, ≥ 10 dB SILK, Vorbis 139 dB) and checks random seeks against the continuous decode. diff --git a/crates/opus/tests/common/mod.rs b/crates/opus/tests/it/common/mod.rs similarity index 86% rename from crates/opus/tests/common/mod.rs rename to crates/opus/tests/it/common/mod.rs index 6c6ec2754..01cc76605 100644 --- a/crates/opus/tests/common/mod.rs +++ b/crates/opus/tests/it/common/mod.rs @@ -32,12 +32,15 @@ fn spectra(x: &[f32], channels: usize, ch: usize, rate: u32) -> Vec<[f64; NFREQ] let w: Vec = (0..win).map(|i| 0.5 - 0.5 * (2.0 * std::f64::consts::PI * (i as f64 + 0.5) / win as f64).cos()).collect(); let nb = NFREQ / scale; let mut out = Vec::with_capacity(nframes); + let tw = twiddles(win); let mut buf = vec![(0f64, 0f64); win]; + let mut spec = vec![(0f64, 0f64); win]; + let mut scratch = vec![(0f64, 0f64); win]; for f in 0..nframes { for i in 0..win { buf[i] = (x[(f * step + i) * channels + ch] as f64 * w[i], 0.0); } - let spec = fft(&buf); + fft(&buf, 1, &mut spec, &tw, &mut scratch); let mut ps = [0f64; NFREQ]; for (k, p) in ps.iter_mut().enumerate().take(nb) { *p = spec[k].0 * spec[k].0 + spec[k].1 * spec[k].1; @@ -47,28 +50,49 @@ fn spectra(x: &[f32], channels: usize, ch: usize, rate: u32) -> Vec<[f64; NFREQ] out } -/// Recursive mixed-radix DFT (sizes with factors 2, 3, 5). -fn fft(x: &[(f64, f64)]) -> Vec<(f64, f64)> { - let n = x.len(); +/// `e^(-2πi·j/n)` for `j` in `0..n`: the twiddle factors of every sub-transform of a size-`n` FFT. +fn twiddles(n: usize) -> Vec<(f64, f64)> { + (0..n) + .map(|j| { + let ang = -2.0 * std::f64::consts::PI * j as f64 / n as f64; + (ang.cos(), ang.sin()) + }) + .collect() +} + +/// Recursive mixed-radix DFT (sizes with factors 2, 3, 5) of `x[0], x[xs], x[2·xs], …` into +/// `out`. `tw` is `twiddles` of the top-level size, which `out.len()` divides; `scratch` holds at +/// least `out.len()` values. +fn fft(x: &[(f64, f64)], xs: usize, out: &mut [(f64, f64)], tw: &[(f64, f64)], scratch: &mut [(f64, f64)]) { + let n = out.len(); if n == 1 { - return x.to_vec(); + out[0] = x[0]; + return; } let p = [2usize, 3, 5].into_iter().find(|p| n.is_multiple_of(*p)).unwrap_or(n); let m = n / p; - let subs: Vec> = (0..p).map(|r| fft(&x.iter().skip(r).step_by(p).copied().collect::>())).collect(); - let mut out = vec![(0f64, 0f64); n]; - for (k, o) in out.iter_mut().enumerate() { + let stride = tw.len() / n; + for r in 0..p { + fft(&x[r * xs..], xs * p, &mut out[r * m..(r + 1) * m], tw, scratch); + } + for (k, o) in scratch[..n].iter_mut().enumerate() { + let j = k % m; let mut acc = (0f64, 0f64); - for (r, sub) in subs.iter().enumerate() { - let v = sub[k % m]; - let ang = -2.0 * std::f64::consts::PI * (r * k) as f64 / n as f64; - let (c, s) = (ang.cos(), ang.sin()); + // e = r·k mod n, kept below n without a division per term. + let mut e = 0; + for r in 0..p { + let v = out[r * m + j]; + let (c, s) = tw[e * stride]; acc.0 += v.0 * c - v.1 * s; acc.1 += v.0 * s + v.1 * c; + e += k; + if e >= n { + e -= n; + } } *o = acc; } - out + out.copy_from_slice(&scratch[..n]); } /// Opus quality metric in the spirit of `opus_compare` (RFC 6716 §6.1): a per-band, diff --git a/crates/opus/tests/it/main.rs b/crates/opus/tests/it/main.rs new file mode 100644 index 000000000..48dc2080a --- /dev/null +++ b/crates/opus/tests/it/main.rs @@ -0,0 +1,4 @@ +mod common; +mod oracle; +mod robustness; +mod vectors; diff --git a/crates/opus/tests/oracle.rs b/crates/opus/tests/it/oracle.rs similarity index 99% rename from crates/opus/tests/oracle.rs rename to crates/opus/tests/it/oracle.rs index 0917479ed..f80236737 100644 --- a/crates/opus/tests/oracle.rs +++ b/crates/opus/tests/it/oracle.rs @@ -7,7 +7,7 @@ //! streams containing SILK differ by the (non-normative) SILK resampler, so they are held to a //! lower SNR bound and the spectral quality metric. -mod common; +use crate::common; use std::process::Command; diff --git a/crates/opus/tests/robustness.rs b/crates/opus/tests/it/robustness.rs similarity index 100% rename from crates/opus/tests/robustness.rs rename to crates/opus/tests/it/robustness.rs diff --git a/crates/opus/tests/vectors.rs b/crates/opus/tests/it/vectors.rs similarity index 99% rename from crates/opus/tests/vectors.rs rename to crates/opus/tests/it/vectors.rs index e36529b7c..7b36fbe7a 100644 --- a/crates/opus/tests/vectors.rs +++ b/crates/opus/tests/it/vectors.rs @@ -7,7 +7,7 @@ //! `.dec` output using our implementation of the opus_compare-style quality metric //! (see `common::opus_quality`), for stereo (`.dec`) and mono (`m.dec`) output at 48 kHz. -mod common; +use crate::common; use std::path::PathBuf; diff --git a/crates/platform/README.md b/crates/platform/README.md index b78c57735..00371ba97 100644 --- a/crates/platform/README.md +++ b/crates/platform/README.md @@ -289,18 +289,18 @@ level 4 High tier, which many hardware decoders refuse; it is now level 4.1 Main | test | what | |---|---| -| `tests/videotoolbox.rs` (macOS) | H.264 High, HEVC Main (open GOP: CRA + RASL) and HEVC Main 10, 640×360 (coded 368: cropping) with B-frames: every picture **bit-exact** with our software decoder, same pts order, count, colour and aspect, also after `reset` + reseek to every later sync sample, mid-stream `flush`, and a full pass after resets; forced mid-stream failures (`VtDecoder::fail_after`) at five points continue with the software decoder's exact output; seeded mutation of samples and parameter sets (bit flips, truncation, corrupt length prefixes) never panics or hangs; HEVC 4:2:2 10-bit is bit-exact with ffmpeg's decode | -| `tests/fallback.rs` (every OS) | `HybridDecoder` with a stand-in hardware decoder failing after N samples (every sync sample ± a few, first / last sample, after a seek): output identical to the software decoder; in-band parameter sets identical to the sample entry's stay in hardware, different ones switch to software | +| `tests/it/videotoolbox.rs` (macOS) | H.264 High, HEVC Main (open GOP: CRA + RASL) and HEVC Main 10, 640×360 (coded 368: cropping) with B-frames: every picture **bit-exact** with our software decoder, same pts order, count, colour and aspect, also after `reset` + reseek to every later sync sample, mid-stream `flush`, and a full pass after resets; forced mid-stream failures (`VtDecoder::fail_after`) at five points continue with the software decoder's exact output; seeded mutation of samples and parameter sets (bit flips, truncation, corrupt length prefixes) never panics or hangs; HEVC 4:2:2 10-bit is bit-exact with ffmpeg's decode | +| `tests/it/fallback.rs` (every OS) | `HybridDecoder` with a stand-in hardware decoder failing after N samples (every sync sample ± a few, first / last sample, after a seek): output identical to the software decoder; in-band parameter sets identical to the sample entry's stay in hardware, different ones switch to software | | `tests/setting.rs` | Hardware decoding Off gives the software decoder through `make_video_decoder` and the media stack (no hardware frames); Auto gives VideoToolbox where available | -| `tests/hardware_encode.rs` (macOS) | what the hardware path takes and declines; round trip through our software decoder (every picture, in order, luma PSNR above 30 dB, keyframes no further apart than asked, no composition offsets); an export through `filmcraft_export` that decodes in our decoder and in ffmpeg / ffprobe (profile, size, frame count, BT.709); the built-in encoder still exporting everything hardware declines; exact output size at sizes that are not multiples of 16; hostile configurations (zero, huge, odd sizes, frame rates, bitrates, keyframe intervals, wrong planes) give errors and never panic; encoders dropped at any point do not crash or hang. The same for **HEVC**: Main profile, 8-bit 4:2:0, `hvc1` entry with VPS / SPS / PPS and 4-byte lengths, MP4 and QuickTime, AAC audio, two-pass refused, the format list agreeing with the probe, ffprobe reading `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 | -| `tests/nvenc.rs` (Windows / Linux, NVIDIA) | H.264 from NVENC (1280×720, 6 Mbps, 72 frames) decodes with our decoder at worst 46.9 dB luma PSNR; IDR at 0, 24 and 48; dts / pts right | -| `tests/nvenc_export.rs` (Windows / Linux; fallback also tested without NVIDIA) | Export with hardware encoding against the software encoder through the export pipeline: the two decoded files at worst 54.8 dB luma PSNR; ffmpeg decodes the file without errors; declined cases go to the software encoder; the counters | -| `tests/nvenc_hevc.rs` (Windows, NVIDIA with HEVC) | HEVC from NVENC (1280×720, 6 Mbps, 72 frames, keyframe every 24) decodes with our HEVC decoder at worst 48.9 dB luma / 51.0 dB chroma PSNR; IDR at 0, 24, 48; dts strictly increasing, pts a permutation, `sps_max_num_reorder_pics` within the dts shift; no parameter sets in the samples; the `hvcC`'s profile / tier / level bytes are the SPS's own; VUI BT.709 limited and timing `(1, 24)`; hostile configurations, sizes, planes and encoders dropped mid-stream give errors, never panics; keyframes every 1–2 pictures without B-frames | -| `tests/nvenc_hevc_export.rs` (Windows, NVIDIA with HEVC) | H.265 export through the real pipeline, MP4 and QuickTime, toggle Auto and Off: counters (72 frames, 1 session, 0 declined), our decoder against the software H.264 export at worst 54.8 dB luma PSNR, ffprobe `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 limited, 72 frames, 24/1, keyframes at 0 and 48, start 0, `ffmpeg -xerror` clean; with AAC audio; 1920×1080 and 642×362 cropped back from the coded size; every decline (HDR, analysis pass, two-pass, interlaced, non-square pixels, MXF, sizes over 65535, odd sizes, sizes outside the GPU's limits) an error naming NVENC, counted once; H.264 with hardware encoding Off never touches NVENC | -| `tests/nvenc_rgba_input.rs` (Windows, NVIDIA) | RGBA input, HEVC Main and H.264 High, 1280×720: solid red, green, blue, white, black and grey decode (our decoder) to the exact BT.709 limited codes of `rgba_to_yuv420_8`, within 2; a moving picture at worst 47.7 dB luma / 50.7 dB chroma PSNR against that conversion; Main 10 has no RGBA input; short pictures and planar / RGBA pictures for the other kind of encoder are errors | -| `tests/nvenc_hevc_probe.rs` (Windows) | the HEVC probe in a fresh process: its cost, the cached answer, `available(Hevc)` following it after `register()` (twice) | +| `tests/it/hardware_encode.rs` (macOS) | what the hardware path takes and declines; round trip through our software decoder (every picture, in order, luma PSNR above 30 dB, keyframes no further apart than asked, no composition offsets); an export through `filmcraft_export` that decodes in our decoder and in ffmpeg / ffprobe (profile, size, frame count, BT.709); the built-in encoder still exporting everything hardware declines; exact output size at sizes that are not multiples of 16; hostile configurations (zero, huge, odd sizes, frame rates, bitrates, keyframe intervals, wrong planes) give errors and never panic; encoders dropped at any point do not crash or hang. The same for **HEVC**: Main profile, 8-bit 4:2:0, `hvc1` entry with VPS / SPS / PPS and 4-byte lengths, MP4 and QuickTime, AAC audio, two-pass refused, the format list agreeing with the probe, ffprobe reading `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 | +| `tests/it/nvenc.rs` (Windows / Linux, NVIDIA) | H.264 from NVENC (1280×720, 6 Mbps, 72 frames) decodes with our decoder at worst 46.9 dB luma PSNR; IDR at 0, 24 and 48; dts / pts right | +| `tests/it/nvenc_export.rs` (Windows / Linux; fallback also tested without NVIDIA) | Export with hardware encoding against the software encoder through the export pipeline: the two decoded files at worst 54.8 dB luma PSNR; ffmpeg decodes the file without errors; declined cases go to the software encoder; the counters | +| `tests/it/nvenc_hevc.rs` (Windows, NVIDIA with HEVC) | HEVC from NVENC (1280×720, 6 Mbps, 72 frames, keyframe every 24) decodes with our HEVC decoder at worst 48.9 dB luma / 51.0 dB chroma PSNR; IDR at 0, 24, 48; dts strictly increasing, pts a permutation, `sps_max_num_reorder_pics` within the dts shift; no parameter sets in the samples; the `hvcC`'s profile / tier / level bytes are the SPS's own; VUI BT.709 limited and timing `(1, 24)`; hostile configurations, sizes, planes and encoders dropped mid-stream give errors, never panics; keyframes every 1–2 pictures without B-frames | +| `tests/it/nvenc_hevc_export.rs` (Windows, NVIDIA with HEVC) | H.265 export through the real pipeline, MP4 and QuickTime, toggle Auto and Off: counters (72 frames, 1 session, 0 declined), our decoder against the software H.264 export at worst 54.8 dB luma PSNR, ffprobe `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 limited, 72 frames, 24/1, keyframes at 0 and 48, start 0, `ffmpeg -xerror` clean; with AAC audio; 1920×1080 and 642×362 cropped back from the coded size; every decline (HDR, analysis pass, two-pass, interlaced, non-square pixels, MXF, sizes over 65535, odd sizes, sizes outside the GPU's limits) an error naming NVENC, counted once; H.264 with hardware encoding Off never touches NVENC | +| `tests/it/nvenc_rgba_input.rs` (Windows, NVIDIA) | RGBA input, HEVC Main and H.264 High, 1280×720: solid red, green, blue, white, black and grey decode (our decoder) to the exact BT.709 limited codes of `rgba_to_yuv420_8`, within 2; a moving picture at worst 47.7 dB luma / 50.7 dB chroma PSNR against that conversion; Main 10 has no RGBA input; short pictures and planar / RGBA pictures for the other kind of encoder are errors | +| `tests/it/nvenc_hevc_probe.rs` (Windows) | the HEVC probe in a fresh process: its cost, the cached answer, `available(Hevc)` following it after `register()` (twice) | | `tests/nvenc_hevc_warm.rs` (Windows) | `register()` answers the HEVC and the Main 10 questions on a thread of its own, before anyone asks; the answers are kept and `hdr_available(Hevc)` follows the Main 10 probe | -| `tests/nvenc_hevc_main10.rs` (Windows, NVIDIA with 10-bit HEVC) | HEVC Main 10 in process, PQ and HLG: a float picture (grey ramp to peak white, a moving box, saturated BT.2020 bars) is converted with `rgbf_to_yuv420_10`, encoded and decoded with our HEVC decoder (`Yuv16`, 10 bits, codes below 1024): worst 71.1 dB luma / 70.6 dB chroma PSNR on the 10-bit scale, 803 distinct luma levels on a ramp row, peak code 940 survives; IDR at 0 and 24; the VUI is BT.2020 / 16 or 18 / BT.2020 NCL / limited; the IDR samples carry two prefix SEI NAL units whose messages are byte-for-byte the expected 137 / 144 payloads (HLG: none; other pictures: none); Main and Main 10 refuse each other's pictures; hostile sizes, bitrates, signals, huge or empty SEI payloads, encoders dropped with SEI in flight | +| `tests/it/nvenc_hevc_main10.rs` (Windows, NVIDIA with 10-bit HEVC) | HEVC Main 10 in process, PQ and HLG: a float picture (grey ramp to peak white, a moving box, saturated BT.2020 bars) is converted with `rgbf_to_yuv420_10`, encoded and decoded with our HEVC decoder (`Yuv16`, 10 bits, codes below 1024): worst 71.1 dB luma / 70.6 dB chroma PSNR on the 10-bit scale, 803 distinct luma levels on a ramp row, peak code 940 survives; IDR at 0 and 24; the VUI is BT.2020 / 16 or 18 / BT.2020 NCL / limited; the IDR samples carry two prefix SEI NAL units whose messages are byte-for-byte the expected 137 / 144 payloads (HLG: none; other pictures: none); Main and Main 10 refuse each other's pictures; hostile sizes, bitrates, signals, huge or empty SEI payloads, encoders dropped with SEI in flight | | `tests/nvenc_hevc_hdr_export.rs` (Windows, NVIDIA with 10-bit HEVC) | HDR sequences through the real export job (PQ MP4, HLG MP4, PQ QuickTime): 24 frames, 1 session, 0 declined; ffprobe `hevc` / `Main 10` / `hvc1` / `yuv420p10le`, `bt2020` / `smpte2084` or `arib-std-b67` / `bt2020nc` / `tv`; PQ first-frame side data Mastering display (max 10000000/10000, min 1/10000) and Content light level, HLG none; `ffmpeg -xerror` clean; ffmpeg's yuv420p10le picture equals our decoder's, code for code; our importer reads PQ / HLG, BT.2020, 10-bit, and the `colr` / `mdcv` / `clli` boxes; luma PSNR against a ProRes HDR export of the same sequence 62.6 dB (PQ) / 61.8 dB (HLG); `settings.sdr` and SDR sequences stay Main 8-bit BT.709 with no HDR boxes; the factory rejects mismatched or wrong-length pictures and clamps NaN / infinity | | `src/nvenc/abi_tests.rs` (Windows / Linux) | FFI structs' sizes, alignments, field offsets, constants, GUIDs and the bit-field masks of the `flags` words (H.264, HEVC, and the HEVC picture parameters with the SEI payload array) against a C compiler's view of NVIDIA's `nvEncodeAPI.h` (12.1) | | `src/nvenc/{mod,hevc,export}.rs` unit tests (Windows) | HEVC NAL types, stripping and parameter-set splitting (including `IDR_N_LP`, whose header byte reads as a PPS in H.264, and one-byte or truncated NAL units); the `hvcC` built from real VPS / SPS / PPS (Main and Main 10: accepted as what was asked for, refused as the other, other depths refused), every truncation and bit flip of the SPS, wrong profile, size and NAL types; HEVC levels; P010 layout (shift, little-endian, interleaving, pitch in bytes, narrow pitches and short buffers); the 10-bit capability decline; the SEI messages | diff --git a/crates/platform/tests/common/mod.rs b/crates/platform/tests/it/common/mod.rs similarity index 96% rename from crates/platform/tests/common/mod.rs rename to crates/platform/tests/it/common/mod.rs index e4f39c59e..74f11bdae 100644 --- a/crates/platform/tests/common/mod.rs +++ b/crates/platform/tests/it/common/mod.rs @@ -9,6 +9,10 @@ use filmcraft_codecs::{DecodedFrame, VideoDecoder}; use filmcraft_frame::PixelData; use filmcraft_isobmff::SampleEntry; +/// NVENC sessions are a limited resource on consumer GPUs, and the encoder counters are +/// process-wide: one NVENC test at a time across every module of the binary. +pub static NVENC_SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// The parity fixtures: (file, ffmpeg arguments). B-frames everywhere (reordering), a 360-line /// picture (coded 368: cropping), two GOPs or more (seeks), and an open-GOP HEVC stream (CRA with /// RASL pictures). diff --git a/crates/platform/tests/fallback.rs b/crates/platform/tests/it/fallback.rs similarity index 84% rename from crates/platform/tests/fallback.rs rename to crates/platform/tests/it/fallback.rs index 8c00e52f5..3b7da8d5c 100644 --- a/crates/platform/tests/fallback.rs +++ b/crates/platform/tests/it/fallback.rs @@ -5,7 +5,7 @@ //! damaged sample that the hardware decoded but the software decoder rejects, inside the replayed //! run, loses no picture. -mod common; +use crate::common; use common::*; use filmcraft_codecs::hw::NalStreamInfo; @@ -51,30 +51,31 @@ fn failures_anywhere_continue_with_the_software_output() { for (name, _) in FIXTURES { let Some(path) = named(&ff, name) else { continue }; let s = read_stream(&path); - let reference = decode_all(filmcraft_codecs::software_video_decoder(&s.entry).unwrap().as_mut(), &s.samples); let syncs: Vec = (0..s.samples.len()).filter(|&i| s.sync[i]).collect(); - let mut points: Vec = vec![0, 1, 3, 7, s.samples.len() - 1, s.samples.len()]; - for &k in &syncs { - points.extend([k, k + 1, k + 2, k + 5]); - } + let [_, k, .., last] = syncs[..] else { panic!("{name}: three GOPs or more") }; + // The stream up to a few samples into its last GOP: past the last failure point the + // hybrid only passes the software decoder's output on. + let samples = &s.samples[..(last + 6).min(s.samples.len())]; + let reference = decode_all(filmcraft_codecs::software_video_decoder(&s.entry).unwrap().as_mut(), samples); + // The hybrid's paths differ by what the replay log holds and what the hardware had + // pending: in the first GOP (nothing out yet, then pictures held for reordering), at and + // just after the second restart point (the log starts over, or an HEVC CRA keeps the GOP + // before it), at the last one (a CRA drops the GOP before the previous one), at the last + // sample and never. Failing anywhere else takes one of those paths. + let points = [0, 1, 3, 7, k, k + 1, k + 2, k + 5, last, last + 2, samples.len() - 1, samples.len()]; for fail_after in points { let mut d = hybrid(&s, fail_after); - let out = decode_all(&mut d, &s.samples); - assert_eq!(d.is_hardware(), fail_after >= s.samples.len(), "{name}: fell back at {fail_after}"); + let out = decode_all(&mut d, samples); + assert_eq!(d.is_hardware(), fail_after >= samples.len(), "{name}: fell back at {fail_after}"); assert_same(&format!("{name} failing after {fail_after} samples"), &out, &reference); } // a failure after a seek (reset) into the second GOP - if let Some(&k) = syncs.get(1) { - let tail = &s.samples[k..]; - let expect = { - let mut sw = filmcraft_codecs::software_video_decoder(&s.entry).unwrap(); - decode_all(sw.as_mut(), tail) - }; - let mut d = hybrid(&s, k + 4); - decode_all(&mut d, &s.samples[..k]); - d.reset(); - assert_same(&format!("{name} failing after a seek"), &decode_all(&mut d, tail), &expect); - } + let tail = &samples[k..]; + let expect = decode_all(filmcraft_codecs::software_video_decoder(&s.entry).unwrap().as_mut(), tail); + let mut d = hybrid(&s, k + 4); + decode_all(&mut d, &samples[..k]); + d.reset(); + assert_same(&format!("{name} failing after a seek"), &decode_all(&mut d, tail), &expect); } } @@ -179,7 +180,9 @@ fn a_sample_software_rejects_during_the_replay_loses_no_pictures() { // the second restart point, and a disposable picture before it that is not the first sample let Some(next) = (1..s.samples.len()).find(|&i| s.sync[i]) else { continue }; let Some(d) = (2..next.saturating_sub(2)).find(|&i| info.is_disposable(&s.samples[i].0)) else { continue }; - let mut stream = s.samples.clone(); + // the first two GOPs: the replay and the restart after it + let end = (next + 1..s.samples.len()).find(|&i| s.sync[i]).unwrap_or(s.samples.len()); + let mut stream = s.samples[..end].to_vec(); stream[d].0 = damaged(&s.samples[d].0, header); let (reference, rejected) = decode_lenient(filmcraft_codecs::software_video_decoder(&s.entry).unwrap().as_mut(), &stream); assert_eq!(rejected, [d], "{name}: the software decoder rejects the damaged sample (and only it)"); diff --git a/crates/platform/tests/hardware_encode.rs b/crates/platform/tests/it/hardware_encode.rs similarity index 100% rename from crates/platform/tests/hardware_encode.rs rename to crates/platform/tests/it/hardware_encode.rs diff --git a/crates/platform/tests/hevc_probe.rs b/crates/platform/tests/it/hevc_probe.rs similarity index 100% rename from crates/platform/tests/hevc_probe.rs rename to crates/platform/tests/it/hevc_probe.rs diff --git a/crates/platform/tests/it/main.rs b/crates/platform/tests/it/main.rs new file mode 100644 index 000000000..b863c7d04 --- /dev/null +++ b/crates/platform/tests/it/main.rs @@ -0,0 +1,13 @@ +mod common; +mod fallback; +mod hardware_encode; +mod hevc_probe; +mod media_foundation; +mod nvenc; +mod nvenc_export; +mod nvenc_hevc; +mod nvenc_hevc_export; +mod nvenc_hevc_main10; +mod nvenc_hevc_probe; +mod nvenc_rgba_input; +mod videotoolbox; diff --git a/crates/platform/tests/media_foundation.rs b/crates/platform/tests/it/media_foundation.rs similarity index 99% rename from crates/platform/tests/media_foundation.rs rename to crates/platform/tests/it/media_foundation.rs index 31c53a7b7..5b7a09d7f 100644 --- a/crates/platform/tests/media_foundation.rs +++ b/crates/platform/tests/it/media_foundation.rs @@ -6,7 +6,7 @@ //! hang. Skips without ffmpeg (fixture generator) or without a hardware decoder. #![cfg(target_os = "windows")] -mod common; +use crate::common; use std::time::Duration; diff --git a/crates/platform/tests/nvenc.rs b/crates/platform/tests/it/nvenc.rs similarity index 100% rename from crates/platform/tests/nvenc.rs rename to crates/platform/tests/it/nvenc.rs diff --git a/crates/platform/tests/nvenc_export.rs b/crates/platform/tests/it/nvenc_export.rs similarity index 95% rename from crates/platform/tests/nvenc_export.rs rename to crates/platform/tests/it/nvenc_export.rs index e0d32a20c..d02bcca35 100644 --- a/crates/platform/tests/nvenc_export.rs +++ b/crates/platform/tests/it/nvenc_export.rs @@ -13,9 +13,6 @@ use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, Project, SequenceS use filmcraft_render::SourceMap; use filmcraft_time::{FrameRate, Tick, TimeRange}; -/// The encoder counters are process-wide: one test at a time. -static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); - fn project(w: u32, h: u32, frames: i64) -> (Arc, filmcraft_project::ItemId, SourceMap) { let rate = FrameRate::FPS_24; let mut p = Project::new("x"); @@ -79,7 +76,7 @@ fn settings(path: String, hw: HardwareEncoding) -> ExportSettings { #[test] fn hardware_export_matches_the_software_export() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); let (p, seq, m) = project(1280, 720, 72); let soft_path = tmp("soft.mp4"); @@ -117,7 +114,7 @@ fn hardware_export_matches_the_software_export() { #[test] fn what_nvenc_does_not_take_goes_to_the_software_encoder() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); let (p, seq, m) = project(640, 360, 24); let run = |name: &str, f: &dyn Fn(&mut ExportSettings)| { diff --git a/crates/platform/tests/nvenc_hevc.rs b/crates/platform/tests/it/nvenc_hevc.rs similarity index 95% rename from crates/platform/tests/nvenc_hevc.rs rename to crates/platform/tests/it/nvenc_hevc.rs index acd2bee9b..f0aebcc45 100644 --- a/crates/platform/tests/nvenc_hevc.rs +++ b/crates/platform/tests/it/nvenc_hevc.rs @@ -43,9 +43,6 @@ fn config(w: u32, h: u32) -> Config { } } -/// NVENC sessions are a limited resource on consumer GPUs: one test at a time. -static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); - type Sources = Vec<(Vec, Vec, Vec)>; /// `None` (the test skips) only on a machine without an HEVC encoder; on one that has it, a @@ -94,7 +91,7 @@ fn hex(b: &[u8]) -> String { #[test] fn the_stream_decodes_to_the_source() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let (w, h) = (1280u32, 720u32); let Some((enc, packets, sources)) = encode_all(&config(w, h), 72) else { return }; assert_eq!(enc.codec(), Codec::Hevc); @@ -179,7 +176,7 @@ fn the_stream_decodes_to_the_source() { #[test] fn nothing_to_encode_with_is_an_error_not_a_crash() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC here"); return; @@ -222,7 +219,7 @@ fn calm(what: &str, f: impl FnOnce() -> T) -> T { #[test] fn hostile_configurations_give_errors_not_crashes() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC here"); return; @@ -266,7 +263,7 @@ fn hostile_configurations_give_errors_not_crashes() { #[test] fn an_encoder_dropped_mid_stream_releases_everything() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC here"); return; @@ -285,7 +282,7 @@ fn an_encoder_dropped_mid_stream_releases_everything() { #[test] fn high_bitrates_move_up_a_level_and_stay_in_the_main_tier() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); // 1080p30 at a 12 / 18 Mbit/s peak: above level 4's Main tier (12), where NVENC's own choice was // level 4 High tier, which many hardware decoders refuse for (w, h, fps, kbps, max_kbps, level_idc) in @@ -302,7 +299,7 @@ fn high_bitrates_move_up_a_level_and_stay_in_the_main_tier() { #[test] fn a_keyframe_every_one_or_two_pictures_is_written_without_b_frames() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); // NVENC wants a GOP longer than the B-frame pattern: short GOPs drop the B-frames, they do not fail for keyint in [1u32, 2] { let cfg = Config { keyint, ..config(640, 360) }; @@ -324,7 +321,7 @@ fn a_keyframe_every_one_or_two_pictures_is_written_without_b_frames() { #[test] fn vui_timing_matches_integer_and_fractional_frame_rates() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); for fps in [(24, 1), (25, 1), (30_000, 1_001)] { let mut cfg = config(640, 360); cfg.fps = fps; diff --git a/crates/platform/tests/nvenc_hevc_export.rs b/crates/platform/tests/it/nvenc_hevc_export.rs similarity index 97% rename from crates/platform/tests/nvenc_hevc_export.rs rename to crates/platform/tests/it/nvenc_hevc_export.rs index 95f074bb8..5fb79a9f7 100644 --- a/crates/platform/tests/nvenc_hevc_export.rs +++ b/crates/platform/tests/it/nvenc_hevc_export.rs @@ -21,9 +21,6 @@ use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, Project, SequenceS use filmcraft_render::SourceMap; use filmcraft_time::{FrameRate, Tick, TimeRange}; -/// The encoder counters are process-wide: one test at a time. -static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); - fn project(w: u32, h: u32, frames: i64) -> (Arc, filmcraft_project::ItemId, SourceMap) { let rate = FrameRate::FPS_24; let mut p = Project::new("x"); @@ -115,7 +112,7 @@ fn delta(before: filmcraft_export::HwEncodeStats) -> (u64, u64, u64) { #[test] fn hevc_exports_play_everywhere() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); assert_eq!(filmcraft_export::available(Format::Hevc), hevc_available(), "the format is available iff NVENC can encode HEVC"); if !hevc_available() { @@ -173,7 +170,7 @@ fn hevc_exports_play_everywhere() { #[test] fn hevc_export_with_audio() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC here"); @@ -202,7 +199,7 @@ fn hevc_export_with_audio() { #[test] fn sizes_that_are_not_multiples_of_the_coding_block_are_cropped_back() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC here"); @@ -230,7 +227,7 @@ fn sizes_that_are_not_multiples_of_the_coding_block_are_cropped_back() { #[test] fn what_nvenc_cannot_do_is_an_error_that_says_why() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); filmcraft_platform::register(); let rate = FrameRate::FPS_24; let base = hevc(tmp("declined.mp4")); diff --git a/crates/platform/tests/nvenc_hevc_main10.rs b/crates/platform/tests/it/nvenc_hevc_main10.rs similarity index 96% rename from crates/platform/tests/nvenc_hevc_main10.rs rename to crates/platform/tests/it/nvenc_hevc_main10.rs index 76d0c9ccc..1b49795dd 100644 --- a/crates/platform/tests/nvenc_hevc_main10.rs +++ b/crates/platform/tests/it/nvenc_hevc_main10.rs @@ -16,9 +16,6 @@ const W: usize = 1280; const H: usize = 720; const FRAMES: usize = 48; -/// NVENC sessions are a limited resource on consumer GPUs: one test at a time. -static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); - fn config(w: u32, h: u32) -> Config { Config { width: w, @@ -134,7 +131,7 @@ fn hex(b: &[u8]) -> String { } fn round_trip(name: &str, signal: ColorSignal, want_sei: bool) { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let Some((enc, packets, sources)) = encode_all(&config(W as u32, H as u32), &signal, FRAMES) else { return }; assert!(enc.is_ten_bit()); assert_eq!(packets.len(), FRAMES, "{name}: one packet per picture"); @@ -239,7 +236,7 @@ fn calm(what: &str, f: impl FnOnce() -> T) -> T { #[test] fn main_10_and_main_encoders_refuse_each_others_pictures() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_hdr_available() { eprintln!("SKIPPED: no NVENC HEVC Main 10 encoder"); return; @@ -268,7 +265,7 @@ fn main_10_and_main_encoders_refuse_each_others_pictures() { #[test] fn hostile_main_10_configurations_give_errors_not_crashes() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_hdr_available() { eprintln!("SKIPPED: no NVENC HEVC Main 10 encoder"); return; @@ -315,7 +312,7 @@ fn hostile_main_10_configurations_give_errors_not_crashes() { #[test] fn the_probe_follows_the_encoder() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let hdr = hevc_hdr_available(); assert!(!hdr || hevc_available(), "Main 10 implies HEVC"); assert_eq!(hevc_hdr_available(), hdr, "the answer is kept"); diff --git a/crates/platform/tests/nvenc_hevc_probe.rs b/crates/platform/tests/it/nvenc_hevc_probe.rs similarity index 100% rename from crates/platform/tests/nvenc_hevc_probe.rs rename to crates/platform/tests/it/nvenc_hevc_probe.rs diff --git a/crates/platform/tests/nvenc_rgba_input.rs b/crates/platform/tests/it/nvenc_rgba_input.rs similarity index 94% rename from crates/platform/tests/nvenc_rgba_input.rs rename to crates/platform/tests/it/nvenc_rgba_input.rs index 4009f3e68..275b1e227 100644 --- a/crates/platform/tests/nvenc_rgba_input.rs +++ b/crates/platform/tests/it/nvenc_rgba_input.rs @@ -6,9 +6,6 @@ use filmcraft_isobmff::{AvcConfig, SampleEntry}; use filmcraft_platform::nvenc::{Codec, Config, Nvenc, Packet, Profile, Signal, available, hevc_available}; -/// NVENC sessions are a limited resource on consumer GPUs: one test at a time. -static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); - /// A moving test picture: gradient background, a moving box, a little texture. fn moving(w: usize, h: usize, i: usize) -> Vec { let mut v = vec![0u8; w * h * 4]; @@ -122,7 +119,7 @@ fn check(profile: Profile) { #[test] fn hevc_rgba_input_converts_like_the_export() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC encoder"); return; @@ -132,7 +129,7 @@ fn hevc_rgba_input_converts_like_the_export() { #[test] fn h264_rgba_input_converts_like_the_export() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !available() { eprintln!("SKIPPED: no NVENC"); return; @@ -142,7 +139,7 @@ fn h264_rgba_input_converts_like_the_export() { #[test] fn main10_has_no_rgba_input_and_short_pictures_are_errors() { - let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _one = crate::common::NVENC_SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if !hevc_available() { eprintln!("SKIPPED: no NVENC HEVC encoder"); return; diff --git a/crates/platform/tests/videotoolbox.rs b/crates/platform/tests/it/videotoolbox.rs similarity index 99% rename from crates/platform/tests/videotoolbox.rs rename to crates/platform/tests/it/videotoolbox.rs index c2ce2f6ae..2db339b1b 100644 --- a/crates/platform/tests/videotoolbox.rs +++ b/crates/platform/tests/it/videotoolbox.rs @@ -6,7 +6,7 @@ //! hang. Skips without ffmpeg (fixture generator) or without a hardware decoder. #![cfg(target_os = "macos")] -mod common; +use crate::common; use std::time::Duration; diff --git a/crates/platform/tests/media_stack.rs b/crates/platform/tests/media_stack.rs index a7d244f5d..e8be399d8 100644 --- a/crates/platform/tests/media_stack.rs +++ b/crates/platform/tests/media_stack.rs @@ -2,6 +2,7 @@ //! decoding Auto and Off (Windows). Its own test binary: the switch is process-wide. #![cfg(target_os = "windows")] +#[path = "it/common/mod.rs"] mod common; use common::*; diff --git a/crates/platform/tests/nvdec.rs b/crates/platform/tests/nvdec.rs index d87f15611..ef1a689dc 100644 --- a/crates/platform/tests/nvdec.rs +++ b/crates/platform/tests/nvdec.rs @@ -4,9 +4,11 @@ //! `reset` + reseek; a forced mid-stream failure must continue in software with the software //! decoder's exact output; what the hardware path does not take is declined; damaged samples give //! errors or fall back, never crash or hang. Skips without ffmpeg (fixture generator) or without -//! an NVIDIA GPU. +//! an NVIDIA GPU. Its own test binary: one test turns the process-wide Hardware decoding switch +//! off and on. #![cfg(all(target_os = "linux", target_pointer_width = "64"))] +#[path = "it/common/mod.rs"] mod common; use std::time::Duration; diff --git a/crates/platform/tests/setting.rs b/crates/platform/tests/setting.rs index c58f4a953..259a02c31 100644 --- a/crates/platform/tests/setting.rs +++ b/crates/platform/tests/setting.rs @@ -2,6 +2,7 @@ //! Off gives the software decoder (and no hardware frames), Auto the hardware one where the //! system has it. Its own test binary: the switch is process-wide. +#[path = "it/common/mod.rs"] mod common; use common::*; diff --git a/crates/platform/tests/vaapi.rs b/crates/platform/tests/vaapi.rs index 507e7768c..f7c140661 100644 --- a/crates/platform/tests/vaapi.rs +++ b/crates/platform/tests/vaapi.rs @@ -8,9 +8,11 @@ //! continue in software with the software decoder's exact output; what the hardware path does not //! take (interlaced and 10-bit H.264, 4:2:2 HEVC) is declined; damaged samples and parameter sets give //! errors or fall back, never crash or hang. Skips without ffmpeg (fixture generator) or without a -//! VA-API driver. +//! VA-API driver. Its own test binary: one test turns the process-wide Hardware decoding switch off +//! and on. #![cfg(target_os = "linux")] +#[path = "it/common/mod.rs"] mod common; use std::time::Duration; diff --git a/crates/platform/tests/vaapi_displays.rs b/crates/platform/tests/vaapi_displays.rs index 8ec45334a..acdb721d8 100644 --- a/crates/platform/tests/vaapi_displays.rs +++ b/crates/platform/tests/vaapi_displays.rs @@ -7,6 +7,7 @@ //! Skips without ffmpeg (fixture generator) or without a VA-API driver. #![cfg(target_os = "linux")] +#[path = "it/common/mod.rs"] mod common; use common::*; diff --git a/crates/prores/README.md b/crates/prores/README.md index 99336e2bc..3659d1a0f 100644 --- a/crates/prores/README.md +++ b/crates/prores/README.md @@ -58,7 +58,7 @@ does), so the decoder doesn't need to know. rayon task directly into the output planes. - Robustness: every size field is validated before allocation, codewords longer than 56 bits are rejected, and arithmetic is overflow-safe. Mutated streams return `Err`, never panic (see - `tests/fuzz.rs`). + `tests/it/fuzz.rs`). ## Encoder @@ -79,7 +79,7 @@ does), so the decoder doesn't need to know. ## Accuracy (vs ffmpeg as an external oracle) -`tests/oracle_decode.rs` decodes 29 fixtures made by ffmpeg's `prores_ks` and `prores_aw` +`tests/it/oracle_decode.rs` decodes 29 fixtures made by ffmpeg's `prores_ks` and `prores_aw` encoders. They cover every profile, testsrc2/mandelbrot/SMPTE bars at 1920×1080, 1918×1080, 1280×720, 640×360, 352×288, 70×38 and 333×201, interlaced 720×486 TFF/BFF and 1080i, 2-MB slices, and 4444 with 8- and 16-bit alpha. The results are compared with ffmpeg's decode at the native @@ -89,7 +89,7 @@ bit-identical (86% for 12-bit XQ mandelbrot). Alpha is always bit-exact. The rem differences are the reference decoder's integer-IDCT rounding. This decoder is closer to the exact IDCT and does not reproduce that approximation. -`tests/oracle_encode.rs` encodes 1920×1080 frames with every profile (testsrc2, mandelbrot and +`tests/it/oracle_encode.rs` encodes 1920×1080 frames with every profile (testsrc2, mandelbrot and noisy content; 4444 with an alpha disc) and writes them into a minimal test-only MOV. ffmpeg then decodes them with `-xerror`, with no errors, and agrees with this decoder within ±1. HQ PSNR is 69.9/69.2 dB (Y/C) on mandelbrot at 100% of the target rate. Busy content lands at 99.9–100.4% @@ -97,7 +97,7 @@ of the target. ## Performance -Measured with `cargo test --release -p filmcraft-prores --test perf -- --ignored --nocapture` on +Measured with `cargo test --release -p filmcraft-prores --test it perf:: -- --ignored --nocapture` on an M4 Pro (14 cores) that was heavily loaded by other jobs, so these are lower bounds. A 1080p HQ frame at the full nominal rate (918 KB) decoded at up to 318 fps threaded and 53 fps on one thread. A prores_ks HQ mandelbrot frame (657 KB) decoded at 390 fps threaded. Encoding a 1080p diff --git a/crates/prores/tests/common/mod.rs b/crates/prores/tests/it/common/mod.rs similarity index 100% rename from crates/prores/tests/common/mod.rs rename to crates/prores/tests/it/common/mod.rs diff --git a/crates/prores/tests/fuzz.rs b/crates/prores/tests/it/fuzz.rs similarity index 99% rename from crates/prores/tests/fuzz.rs rename to crates/prores/tests/it/fuzz.rs index cd012a04c..4b4292f8a 100644 --- a/crates/prores/tests/fuzz.rs +++ b/crates/prores/tests/it/fuzz.rs @@ -1,6 +1,6 @@ //! Robustness: mutated and random inputs must never panic (errors are fine). -mod common; +use crate::common; use filmcraft_prores::*; struct Rng(u64); diff --git a/crates/prores/tests/fuzz_hostile.rs b/crates/prores/tests/it/fuzz_hostile.rs similarity index 100% rename from crates/prores/tests/fuzz_hostile.rs rename to crates/prores/tests/it/fuzz_hostile.rs diff --git a/crates/prores/tests/it/main.rs b/crates/prores/tests/it/main.rs new file mode 100644 index 000000000..f6423aad1 --- /dev/null +++ b/crates/prores/tests/it/main.rs @@ -0,0 +1,7 @@ +mod common; +mod fuzz; +mod fuzz_hostile; +mod oracle_decode; +mod oracle_encode; +mod perf; +mod roundtrip; diff --git a/crates/prores/tests/oracle_decode.rs b/crates/prores/tests/it/oracle_decode.rs similarity index 99% rename from crates/prores/tests/oracle_decode.rs rename to crates/prores/tests/it/oracle_decode.rs index 1b93eda97..660aceb52 100644 --- a/crates/prores/tests/oracle_decode.rs +++ b/crates/prores/tests/it/oracle_decode.rs @@ -1,6 +1,6 @@ //! Decoder accuracy against ffmpeg's ProRes decoder (oracle only; skipped without ffmpeg). -mod common; +use crate::common; use common::*; use filmcraft_prores::{AlphaType, ChromaFormat, DecodeOptions, Interlace, decode_frame, decode_frame_with, probe}; diff --git a/crates/prores/tests/oracle_encode.rs b/crates/prores/tests/it/oracle_encode.rs similarity index 99% rename from crates/prores/tests/oracle_encode.rs rename to crates/prores/tests/it/oracle_encode.rs index ed269a6d3..5fd9adc9f 100644 --- a/crates/prores/tests/oracle_encode.rs +++ b/crates/prores/tests/it/oracle_encode.rs @@ -1,6 +1,6 @@ //! Encoder conformance and quality against ffmpeg's ProRes decoder (oracle only). -mod common; +use crate::common; use common::*; use filmcraft_prores::{ChromaFormat, DecodeOptions, Encoder, Frame, Profile, decode_frame, decode_frame_with}; diff --git a/crates/prores/tests/perf.rs b/crates/prores/tests/it/perf.rs similarity index 95% rename from crates/prores/tests/perf.rs rename to crates/prores/tests/it/perf.rs index a35e12828..1c9902b93 100644 --- a/crates/prores/tests/perf.rs +++ b/crates/prores/tests/it/perf.rs @@ -1,7 +1,7 @@ //! Decode/encode throughput. Run with -//! `cargo test --release -p filmcraft-prores --test perf -- --ignored --nocapture`. +//! `cargo test --release -p filmcraft-prores --test it perf:: -- --ignored --nocapture`. -mod common; +use crate::common; use filmcraft_prores::*; use std::time::Instant; diff --git a/crates/prores/tests/roundtrip.rs b/crates/prores/tests/it/roundtrip.rs similarity index 99% rename from crates/prores/tests/roundtrip.rs rename to crates/prores/tests/it/roundtrip.rs index e3b058e0b..b6ced3ba5 100644 --- a/crates/prores/tests/roundtrip.rs +++ b/crates/prores/tests/it/roundtrip.rs @@ -1,6 +1,6 @@ //! Encoder → decoder round trips on synthetic content (no external tools needed). -mod common; +use crate::common; use common::psnr; use filmcraft_prores::*; diff --git a/crates/render/tests/it/main.rs b/crates/render/tests/it/main.rs new file mode 100644 index 000000000..73e898571 --- /dev/null +++ b/crates/render/tests/it/main.rs @@ -0,0 +1,4 @@ +mod median_radius; +mod noise_frame_cadence; +mod sharpen_preview_res; +mod underline_direction; diff --git a/crates/render/tests/median_radius.rs b/crates/render/tests/it/median_radius.rs similarity index 100% rename from crates/render/tests/median_radius.rs rename to crates/render/tests/it/median_radius.rs diff --git a/crates/render/tests/noise_frame_cadence.rs b/crates/render/tests/it/noise_frame_cadence.rs similarity index 100% rename from crates/render/tests/noise_frame_cadence.rs rename to crates/render/tests/it/noise_frame_cadence.rs diff --git a/crates/render/tests/sharpen_preview_res.rs b/crates/render/tests/it/sharpen_preview_res.rs similarity index 100% rename from crates/render/tests/sharpen_preview_res.rs rename to crates/render/tests/it/sharpen_preview_res.rs diff --git a/crates/render/tests/underline_direction.rs b/crates/render/tests/it/underline_direction.rs similarity index 100% rename from crates/render/tests/underline_direction.rs rename to crates/render/tests/it/underline_direction.rs diff --git a/crates/ui-egui/tests/audio51_ui.rs b/crates/ui-egui/tests/it/audio51_ui.rs similarity index 100% rename from crates/ui-egui/tests/audio51_ui.rs rename to crates/ui-egui/tests/it/audio51_ui.rs diff --git a/crates/ui-egui/tests/audio_fx_editor_ui.rs b/crates/ui-egui/tests/it/audio_fx_editor_ui.rs similarity index 100% rename from crates/ui-egui/tests/audio_fx_editor_ui.rs rename to crates/ui-egui/tests/it/audio_fx_editor_ui.rs diff --git a/crates/ui-egui/tests/caption_properties_ui.rs b/crates/ui-egui/tests/it/caption_properties_ui.rs similarity index 100% rename from crates/ui-egui/tests/caption_properties_ui.rs rename to crates/ui-egui/tests/it/caption_properties_ui.rs diff --git a/crates/ui-egui/tests/captions_font_ui.rs b/crates/ui-egui/tests/it/captions_font_ui.rs similarity index 100% rename from crates/ui-egui/tests/captions_font_ui.rs rename to crates/ui-egui/tests/it/captions_font_ui.rs diff --git a/crates/ui-egui/tests/chrome_ui.rs b/crates/ui-egui/tests/it/chrome_ui.rs similarity index 100% rename from crates/ui-egui/tests/chrome_ui.rs rename to crates/ui-egui/tests/it/chrome_ui.rs diff --git a/crates/ui-egui/tests/clip_audio_ui.rs b/crates/ui-egui/tests/it/clip_audio_ui.rs similarity index 100% rename from crates/ui-egui/tests/clip_audio_ui.rs rename to crates/ui-egui/tests/it/clip_audio_ui.rs diff --git a/crates/ui-egui/tests/clip_dialogs_ui.rs b/crates/ui-egui/tests/it/clip_dialogs_ui.rs similarity index 100% rename from crates/ui-egui/tests/clip_dialogs_ui.rs rename to crates/ui-egui/tests/it/clip_dialogs_ui.rs diff --git a/crates/ui-egui/tests/clipboard_paste_ui.rs b/crates/ui-egui/tests/it/clipboard_paste_ui.rs similarity index 100% rename from crates/ui-egui/tests/clipboard_paste_ui.rs rename to crates/ui-egui/tests/it/clipboard_paste_ui.rs diff --git a/crates/ui-egui/tests/color_ui.rs b/crates/ui-egui/tests/it/color_ui.rs similarity index 100% rename from crates/ui-egui/tests/color_ui.rs rename to crates/ui-egui/tests/it/color_ui.rs diff --git a/crates/ui-egui/tests/delete_caption_tracks_ui.rs b/crates/ui-egui/tests/it/delete_caption_tracks_ui.rs similarity index 100% rename from crates/ui-egui/tests/delete_caption_tracks_ui.rs rename to crates/ui-egui/tests/it/delete_caption_tracks_ui.rs diff --git a/crates/ui-egui/tests/deselect_ui.rs b/crates/ui-egui/tests/it/deselect_ui.rs similarity index 100% rename from crates/ui-egui/tests/deselect_ui.rs rename to crates/ui-egui/tests/it/deselect_ui.rs diff --git a/crates/ui-egui/tests/dialog_buttons_ui.rs b/crates/ui-egui/tests/it/dialog_buttons_ui.rs similarity index 100% rename from crates/ui-egui/tests/dialog_buttons_ui.rs rename to crates/ui-egui/tests/it/dialog_buttons_ui.rs diff --git a/crates/ui-egui/tests/edit_point_menu_ui.rs b/crates/ui-egui/tests/it/edit_point_menu_ui.rs similarity index 100% rename from crates/ui-egui/tests/edit_point_menu_ui.rs rename to crates/ui-egui/tests/it/edit_point_menu_ui.rs diff --git a/crates/ui-egui/tests/escape_drag_ui.rs b/crates/ui-egui/tests/it/escape_drag_ui.rs similarity index 100% rename from crates/ui-egui/tests/escape_drag_ui.rs rename to crates/ui-egui/tests/it/escape_drag_ui.rs diff --git a/crates/ui-egui/tests/essential_sound_ui.rs b/crates/ui-egui/tests/it/essential_sound_ui.rs similarity index 100% rename from crates/ui-egui/tests/essential_sound_ui.rs rename to crates/ui-egui/tests/it/essential_sound_ui.rs diff --git a/crates/ui-egui/tests/export_menu_ui.rs b/crates/ui-egui/tests/it/export_menu_ui.rs similarity index 100% rename from crates/ui-egui/tests/export_menu_ui.rs rename to crates/ui-egui/tests/it/export_menu_ui.rs diff --git a/crates/ui-egui/tests/export_ui.rs b/crates/ui-egui/tests/it/export_ui.rs similarity index 100% rename from crates/ui-egui/tests/export_ui.rs rename to crates/ui-egui/tests/it/export_ui.rs diff --git a/crates/ui-egui/tests/frame_export_ui.rs b/crates/ui-egui/tests/it/frame_export_ui.rs similarity index 100% rename from crates/ui-egui/tests/frame_export_ui.rs rename to crates/ui-egui/tests/it/frame_export_ui.rs diff --git a/crates/ui-egui/tests/graphics_handles_ui.rs b/crates/ui-egui/tests/it/graphics_handles_ui.rs similarity index 100% rename from crates/ui-egui/tests/graphics_handles_ui.rs rename to crates/ui-egui/tests/it/graphics_handles_ui.rs diff --git a/crates/ui-egui/tests/graphics_notice_ui.rs b/crates/ui-egui/tests/it/graphics_notice_ui.rs similarity index 100% rename from crates/ui-egui/tests/graphics_notice_ui.rs rename to crates/ui-egui/tests/it/graphics_notice_ui.rs diff --git a/crates/ui-egui/tests/graphics_templates_ui.rs b/crates/ui-egui/tests/it/graphics_templates_ui.rs similarity index 100% rename from crates/ui-egui/tests/graphics_templates_ui.rs rename to crates/ui-egui/tests/it/graphics_templates_ui.rs diff --git a/crates/ui-egui/tests/graphics_ui.rs b/crates/ui-egui/tests/it/graphics_ui.rs similarity index 100% rename from crates/ui-egui/tests/graphics_ui.rs rename to crates/ui-egui/tests/it/graphics_ui.rs diff --git a/crates/ui-egui/tests/interchange_export_ui.rs b/crates/ui-egui/tests/it/interchange_export_ui.rs similarity index 100% rename from crates/ui-egui/tests/interchange_export_ui.rs rename to crates/ui-egui/tests/it/interchange_export_ui.rs diff --git a/crates/ui-egui/tests/interface_zoom_ui.rs b/crates/ui-egui/tests/it/interface_zoom_ui.rs similarity index 100% rename from crates/ui-egui/tests/interface_zoom_ui.rs rename to crates/ui-egui/tests/it/interface_zoom_ui.rs diff --git a/crates/ui-egui/tests/keyboard_ui.rs b/crates/ui-egui/tests/it/keyboard_ui.rs similarity index 100% rename from crates/ui-egui/tests/keyboard_ui.rs rename to crates/ui-egui/tests/it/keyboard_ui.rs diff --git a/crates/ui-egui/tests/keyframes_ui.rs b/crates/ui-egui/tests/it/keyframes_ui.rs similarity index 100% rename from crates/ui-egui/tests/keyframes_ui.rs rename to crates/ui-egui/tests/it/keyframes_ui.rs diff --git a/crates/ui-egui/tests/links_ui.rs b/crates/ui-egui/tests/it/links_ui.rs similarity index 100% rename from crates/ui-egui/tests/links_ui.rs rename to crates/ui-egui/tests/it/links_ui.rs diff --git a/crates/ui-egui/tests/it/main.rs b/crates/ui-egui/tests/it/main.rs new file mode 100644 index 000000000..c5a3437d2 --- /dev/null +++ b/crates/ui-egui/tests/it/main.rs @@ -0,0 +1,80 @@ +mod audio51_ui; +mod audio_fx_editor_ui; +mod caption_properties_ui; +mod captions_font_ui; +mod chrome_ui; +mod clip_audio_ui; +mod clip_dialogs_ui; +mod clipboard_paste_ui; +mod color_ui; +mod delete_caption_tracks_ui; +mod deselect_ui; +mod dialog_buttons_ui; +mod edit_point_menu_ui; +mod escape_drag_ui; +mod essential_sound_ui; +mod export_menu_ui; +mod export_ui; +mod frame_export_ui; +mod graphics_handles_ui; +mod graphics_notice_ui; +mod graphics_templates_ui; +mod graphics_ui; +mod interchange_export_ui; +mod interface_zoom_ui; +mod keyboard_ui; +mod keyframes_ui; +mod links_ui; +mod marked_playback_position; +mod mask_roundness_ui; +mod masks_ui; +mod media_ui; +mod menus_ui; +mod meters_ui; +mod mixer_ui; +mod monitor_geometry; +mod multicam_ui; +mod native_dialogs_ui; +mod nest_editing_ui; +mod nest_rendering_ui; +mod nested_ui; +mod panels_ui; +mod par_ui; +mod perf_ui; +mod playback_clock_ui; +mod project_panel_ui; +mod properties_scroll_ui; +mod remix_ui; +mod scripted; +mod sequence_tabs_ui; +mod sequence_ui; +mod settings_ui; +mod shared_docking_ui; +mod source_drag_ui; +mod source_playback_ui; +mod timecode_ui; +mod timeline_drop_ui; +mod timeline_escape_ui; +mod timeline_gap_ui; +mod timeline_marquee_ui; +mod timeline_middle_pan_ui; +mod timeline_option_drag_ui; +mod timeline_scroll_ui; +mod timeline_tabs_ui; +mod timeline_track_height_ui; +mod timeline_transition_move_ui; +mod timeline_trim_ui; +mod track_color_ui; +mod track_select_drag_ui; +mod transcript_ui; +mod transform_handles_ui; +mod transition_drop_ui; +mod transitions_ui; +mod tts_ui; +mod ui_control_input; +mod ui_dock_resize; +mod ui_keyboard_focus; +mod unlink_ui; +mod view_ui; +mod voiceover_ui; +mod workspaces_ui; diff --git a/crates/ui-egui/tests/marked_playback_position.rs b/crates/ui-egui/tests/it/marked_playback_position.rs similarity index 100% rename from crates/ui-egui/tests/marked_playback_position.rs rename to crates/ui-egui/tests/it/marked_playback_position.rs diff --git a/crates/ui-egui/tests/mask_roundness_ui.rs b/crates/ui-egui/tests/it/mask_roundness_ui.rs similarity index 100% rename from crates/ui-egui/tests/mask_roundness_ui.rs rename to crates/ui-egui/tests/it/mask_roundness_ui.rs diff --git a/crates/ui-egui/tests/masks_ui.rs b/crates/ui-egui/tests/it/masks_ui.rs similarity index 100% rename from crates/ui-egui/tests/masks_ui.rs rename to crates/ui-egui/tests/it/masks_ui.rs diff --git a/crates/ui-egui/tests/media_ui.rs b/crates/ui-egui/tests/it/media_ui.rs similarity index 100% rename from crates/ui-egui/tests/media_ui.rs rename to crates/ui-egui/tests/it/media_ui.rs diff --git a/crates/ui-egui/tests/menus_ui.rs b/crates/ui-egui/tests/it/menus_ui.rs similarity index 100% rename from crates/ui-egui/tests/menus_ui.rs rename to crates/ui-egui/tests/it/menus_ui.rs diff --git a/crates/ui-egui/tests/meters_ui.rs b/crates/ui-egui/tests/it/meters_ui.rs similarity index 100% rename from crates/ui-egui/tests/meters_ui.rs rename to crates/ui-egui/tests/it/meters_ui.rs diff --git a/crates/ui-egui/tests/mixer_ui.rs b/crates/ui-egui/tests/it/mixer_ui.rs similarity index 100% rename from crates/ui-egui/tests/mixer_ui.rs rename to crates/ui-egui/tests/it/mixer_ui.rs diff --git a/crates/ui-egui/tests/monitor_geometry.rs b/crates/ui-egui/tests/it/monitor_geometry.rs similarity index 100% rename from crates/ui-egui/tests/monitor_geometry.rs rename to crates/ui-egui/tests/it/monitor_geometry.rs diff --git a/crates/ui-egui/tests/multicam_ui.rs b/crates/ui-egui/tests/it/multicam_ui.rs similarity index 100% rename from crates/ui-egui/tests/multicam_ui.rs rename to crates/ui-egui/tests/it/multicam_ui.rs diff --git a/crates/ui-egui/tests/native_dialogs_ui.rs b/crates/ui-egui/tests/it/native_dialogs_ui.rs similarity index 100% rename from crates/ui-egui/tests/native_dialogs_ui.rs rename to crates/ui-egui/tests/it/native_dialogs_ui.rs diff --git a/crates/ui-egui/tests/nest_editing_ui.rs b/crates/ui-egui/tests/it/nest_editing_ui.rs similarity index 100% rename from crates/ui-egui/tests/nest_editing_ui.rs rename to crates/ui-egui/tests/it/nest_editing_ui.rs diff --git a/crates/ui-egui/tests/nest_rendering_ui.rs b/crates/ui-egui/tests/it/nest_rendering_ui.rs similarity index 100% rename from crates/ui-egui/tests/nest_rendering_ui.rs rename to crates/ui-egui/tests/it/nest_rendering_ui.rs diff --git a/crates/ui-egui/tests/nested_ui.rs b/crates/ui-egui/tests/it/nested_ui.rs similarity index 100% rename from crates/ui-egui/tests/nested_ui.rs rename to crates/ui-egui/tests/it/nested_ui.rs diff --git a/crates/ui-egui/tests/panels_ui.rs b/crates/ui-egui/tests/it/panels_ui.rs similarity index 100% rename from crates/ui-egui/tests/panels_ui.rs rename to crates/ui-egui/tests/it/panels_ui.rs diff --git a/crates/ui-egui/tests/par_ui.rs b/crates/ui-egui/tests/it/par_ui.rs similarity index 100% rename from crates/ui-egui/tests/par_ui.rs rename to crates/ui-egui/tests/it/par_ui.rs diff --git a/crates/ui-egui/tests/perf_ui.rs b/crates/ui-egui/tests/it/perf_ui.rs similarity index 100% rename from crates/ui-egui/tests/perf_ui.rs rename to crates/ui-egui/tests/it/perf_ui.rs diff --git a/crates/ui-egui/tests/playback_clock_ui.rs b/crates/ui-egui/tests/it/playback_clock_ui.rs similarity index 100% rename from crates/ui-egui/tests/playback_clock_ui.rs rename to crates/ui-egui/tests/it/playback_clock_ui.rs diff --git a/crates/ui-egui/tests/project_panel_ui.rs b/crates/ui-egui/tests/it/project_panel_ui.rs similarity index 100% rename from crates/ui-egui/tests/project_panel_ui.rs rename to crates/ui-egui/tests/it/project_panel_ui.rs diff --git a/crates/ui-egui/tests/properties_scroll_ui.rs b/crates/ui-egui/tests/it/properties_scroll_ui.rs similarity index 100% rename from crates/ui-egui/tests/properties_scroll_ui.rs rename to crates/ui-egui/tests/it/properties_scroll_ui.rs diff --git a/crates/ui-egui/tests/remix_ui.rs b/crates/ui-egui/tests/it/remix_ui.rs similarity index 100% rename from crates/ui-egui/tests/remix_ui.rs rename to crates/ui-egui/tests/it/remix_ui.rs diff --git a/crates/ui-egui/tests/scripted.rs b/crates/ui-egui/tests/it/scripted.rs similarity index 99% rename from crates/ui-egui/tests/scripted.rs rename to crates/ui-egui/tests/it/scripted.rs index 0bb1dd741..0fc6fbcc0 100644 --- a/crates/ui-egui/tests/scripted.rs +++ b/crates/ui-egui/tests/it/scripted.rs @@ -401,7 +401,7 @@ impl Driver { } /// Renders the Trim Monitor and the Keyboard Shortcuts dialog to PNGs for visual review (needs a -/// GPU): `cargo test -p filmcraft-ui-egui --test scripted -- --ignored screenshots`. +/// GPU): `cargo test -p filmcraft-ui-egui --test it scripted::screenshots -- --ignored`. #[test] #[ignore] fn screenshots() { diff --git a/crates/ui-egui/tests/sequence_tabs_ui.rs b/crates/ui-egui/tests/it/sequence_tabs_ui.rs similarity index 100% rename from crates/ui-egui/tests/sequence_tabs_ui.rs rename to crates/ui-egui/tests/it/sequence_tabs_ui.rs diff --git a/crates/ui-egui/tests/sequence_ui.rs b/crates/ui-egui/tests/it/sequence_ui.rs similarity index 100% rename from crates/ui-egui/tests/sequence_ui.rs rename to crates/ui-egui/tests/it/sequence_ui.rs diff --git a/crates/ui-egui/tests/settings_ui.rs b/crates/ui-egui/tests/it/settings_ui.rs similarity index 100% rename from crates/ui-egui/tests/settings_ui.rs rename to crates/ui-egui/tests/it/settings_ui.rs diff --git a/crates/ui-egui/tests/shared_docking_ui.rs b/crates/ui-egui/tests/it/shared_docking_ui.rs similarity index 100% rename from crates/ui-egui/tests/shared_docking_ui.rs rename to crates/ui-egui/tests/it/shared_docking_ui.rs diff --git a/crates/ui-egui/tests/source_drag_ui.rs b/crates/ui-egui/tests/it/source_drag_ui.rs similarity index 100% rename from crates/ui-egui/tests/source_drag_ui.rs rename to crates/ui-egui/tests/it/source_drag_ui.rs diff --git a/crates/ui-egui/tests/source_playback_ui.rs b/crates/ui-egui/tests/it/source_playback_ui.rs similarity index 100% rename from crates/ui-egui/tests/source_playback_ui.rs rename to crates/ui-egui/tests/it/source_playback_ui.rs diff --git a/crates/ui-egui/tests/timecode_ui.rs b/crates/ui-egui/tests/it/timecode_ui.rs similarity index 100% rename from crates/ui-egui/tests/timecode_ui.rs rename to crates/ui-egui/tests/it/timecode_ui.rs diff --git a/crates/ui-egui/tests/timeline_drop_ui.rs b/crates/ui-egui/tests/it/timeline_drop_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_drop_ui.rs rename to crates/ui-egui/tests/it/timeline_drop_ui.rs diff --git a/crates/ui-egui/tests/timeline_escape_ui.rs b/crates/ui-egui/tests/it/timeline_escape_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_escape_ui.rs rename to crates/ui-egui/tests/it/timeline_escape_ui.rs diff --git a/crates/ui-egui/tests/timeline_gap_ui.rs b/crates/ui-egui/tests/it/timeline_gap_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_gap_ui.rs rename to crates/ui-egui/tests/it/timeline_gap_ui.rs diff --git a/crates/ui-egui/tests/timeline_marquee_ui.rs b/crates/ui-egui/tests/it/timeline_marquee_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_marquee_ui.rs rename to crates/ui-egui/tests/it/timeline_marquee_ui.rs diff --git a/crates/ui-egui/tests/timeline_middle_pan_ui.rs b/crates/ui-egui/tests/it/timeline_middle_pan_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_middle_pan_ui.rs rename to crates/ui-egui/tests/it/timeline_middle_pan_ui.rs diff --git a/crates/ui-egui/tests/timeline_option_drag_ui.rs b/crates/ui-egui/tests/it/timeline_option_drag_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_option_drag_ui.rs rename to crates/ui-egui/tests/it/timeline_option_drag_ui.rs diff --git a/crates/ui-egui/tests/timeline_scroll_ui.rs b/crates/ui-egui/tests/it/timeline_scroll_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_scroll_ui.rs rename to crates/ui-egui/tests/it/timeline_scroll_ui.rs diff --git a/crates/ui-egui/tests/timeline_tabs_ui.rs b/crates/ui-egui/tests/it/timeline_tabs_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_tabs_ui.rs rename to crates/ui-egui/tests/it/timeline_tabs_ui.rs diff --git a/crates/ui-egui/tests/timeline_track_height_ui.rs b/crates/ui-egui/tests/it/timeline_track_height_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_track_height_ui.rs rename to crates/ui-egui/tests/it/timeline_track_height_ui.rs diff --git a/crates/ui-egui/tests/timeline_transition_move_ui.rs b/crates/ui-egui/tests/it/timeline_transition_move_ui.rs similarity index 100% rename from crates/ui-egui/tests/timeline_transition_move_ui.rs rename to crates/ui-egui/tests/it/timeline_transition_move_ui.rs diff --git a/crates/ui-egui/tests/timeline_trim_ui.rs b/crates/ui-egui/tests/it/timeline_trim_ui.rs similarity index 99% rename from crates/ui-egui/tests/timeline_trim_ui.rs rename to crates/ui-egui/tests/it/timeline_trim_ui.rs index 94aabf400..d6243d548 100644 --- a/crates/ui-egui/tests/timeline_trim_ui.rs +++ b/crates/ui-egui/tests/it/timeline_trim_ui.rs @@ -256,7 +256,7 @@ fn cursor_hit_test_and_press_agree_at_a_cut() { } /// Renders the hover bracket at the City_Night_Drive / Desert_Dunes cut to PNGs for visual -/// review (needs a GPU): `cargo test -p filmcraft-ui-egui --test timeline_trim_ui -- --ignored +/// review (needs a GPU): `cargo test -p filmcraft-ui-egui --test it timeline_trim_ui:: -- --ignored /// hover_bracket_screenshots`. Yellow on City_Night_Drive's Out edge just left of the cut and on /// Desert_Dunes' In edge just right of it, red for the Ripple tool, both sides for the Rolling tool. #[test] @@ -301,7 +301,7 @@ fn dragging_a_caption_edge_inward_trims_it() { /// #374: renders the Ambient_Score music clip mid-way through a trim of its In edge, with the /// mouse button still down, for visual review (needs a GPU): `cargo test -p filmcraft-ui-egui -/// --test timeline_trim_ui -- --ignored trim_waveform_screenshots`. The waveform stays where it was +/// --test it timeline_trim_ui::trim_waveform_screenshots -- --ignored`. The waveform stays where it was /// and is cut off at the moving edge; it used to squeeze the whole clip's waveform into the shorter /// clip, so the sound under the pointer moved while dragging. #[test] diff --git a/crates/ui-egui/tests/track_color_ui.rs b/crates/ui-egui/tests/it/track_color_ui.rs similarity index 100% rename from crates/ui-egui/tests/track_color_ui.rs rename to crates/ui-egui/tests/it/track_color_ui.rs diff --git a/crates/ui-egui/tests/track_select_drag_ui.rs b/crates/ui-egui/tests/it/track_select_drag_ui.rs similarity index 100% rename from crates/ui-egui/tests/track_select_drag_ui.rs rename to crates/ui-egui/tests/it/track_select_drag_ui.rs diff --git a/crates/ui-egui/tests/transcript_ui.rs b/crates/ui-egui/tests/it/transcript_ui.rs similarity index 100% rename from crates/ui-egui/tests/transcript_ui.rs rename to crates/ui-egui/tests/it/transcript_ui.rs diff --git a/crates/ui-egui/tests/transform_handles_ui.rs b/crates/ui-egui/tests/it/transform_handles_ui.rs similarity index 100% rename from crates/ui-egui/tests/transform_handles_ui.rs rename to crates/ui-egui/tests/it/transform_handles_ui.rs diff --git a/crates/ui-egui/tests/transition_drop_ui.rs b/crates/ui-egui/tests/it/transition_drop_ui.rs similarity index 100% rename from crates/ui-egui/tests/transition_drop_ui.rs rename to crates/ui-egui/tests/it/transition_drop_ui.rs diff --git a/crates/ui-egui/tests/transitions_ui.rs b/crates/ui-egui/tests/it/transitions_ui.rs similarity index 100% rename from crates/ui-egui/tests/transitions_ui.rs rename to crates/ui-egui/tests/it/transitions_ui.rs diff --git a/crates/ui-egui/tests/tts_ui.rs b/crates/ui-egui/tests/it/tts_ui.rs similarity index 100% rename from crates/ui-egui/tests/tts_ui.rs rename to crates/ui-egui/tests/it/tts_ui.rs diff --git a/crates/ui-egui/tests/ui_control_input.rs b/crates/ui-egui/tests/it/ui_control_input.rs similarity index 100% rename from crates/ui-egui/tests/ui_control_input.rs rename to crates/ui-egui/tests/it/ui_control_input.rs diff --git a/crates/ui-egui/tests/ui_dock_resize.rs b/crates/ui-egui/tests/it/ui_dock_resize.rs similarity index 100% rename from crates/ui-egui/tests/ui_dock_resize.rs rename to crates/ui-egui/tests/it/ui_dock_resize.rs diff --git a/crates/ui-egui/tests/ui_keyboard_focus.rs b/crates/ui-egui/tests/it/ui_keyboard_focus.rs similarity index 100% rename from crates/ui-egui/tests/ui_keyboard_focus.rs rename to crates/ui-egui/tests/it/ui_keyboard_focus.rs diff --git a/crates/ui-egui/tests/unlink_ui.rs b/crates/ui-egui/tests/it/unlink_ui.rs similarity index 100% rename from crates/ui-egui/tests/unlink_ui.rs rename to crates/ui-egui/tests/it/unlink_ui.rs diff --git a/crates/ui-egui/tests/view_ui.rs b/crates/ui-egui/tests/it/view_ui.rs similarity index 100% rename from crates/ui-egui/tests/view_ui.rs rename to crates/ui-egui/tests/it/view_ui.rs diff --git a/crates/ui-egui/tests/voiceover_ui.rs b/crates/ui-egui/tests/it/voiceover_ui.rs similarity index 100% rename from crates/ui-egui/tests/voiceover_ui.rs rename to crates/ui-egui/tests/it/voiceover_ui.rs diff --git a/crates/ui-egui/tests/workspaces_ui.rs b/crates/ui-egui/tests/it/workspaces_ui.rs similarity index 100% rename from crates/ui-egui/tests/workspaces_ui.rs rename to crates/ui-egui/tests/it/workspaces_ui.rs diff --git a/crates/vp8/README.md b/crates/vp8/README.md index affb7ff34..5b855a21f 100644 --- a/crates/vp8/README.md +++ b/crates/vp8/README.md @@ -28,8 +28,8 @@ test oracles. Bit-exact with libvpx on the 62 libvpx VP8 test vectors checked (`vp80-00-comprehensive-001` to `018` and the `vp80-01` to `vp80-06` intra, inter, segmentation, partition, sharpness and small -size vectors; `tests/vectors.rs` checks them when they are present) and on the generated fixtures -of `tests/conformance.rs`. +size vectors; `tests/it/vectors.rs` checks them when they are present) and on the generated fixtures +of `tests/it/conformance.rs`. ## API diff --git a/crates/vp8/tests/common/mod.rs b/crates/vp8/tests/it/common/mod.rs similarity index 94% rename from crates/vp8/tests/common/mod.rs rename to crates/vp8/tests/it/common/mod.rs index cc42e8e88..15416b864 100644 --- a/crates/vp8/tests/common/mod.rs +++ b/crates/vp8/tests/it/common/mod.rs @@ -108,15 +108,17 @@ pub fn run(cmd: &mut Command) -> bool { /// Generate (if needed) the fixture stream and its reference decode by libvpx through ffmpeg. /// `None` (the test skips) without ffmpeg or its libvpx encoder. pub fn ensure(f: &Fixture) -> Option<(PathBuf, PathBuf)> { - // one generation at a time: tests share fixtures, and realtime libvpx encodes differ from run - // to run, so a stream and its reference decode must come from the same encode - static GENERATING: std::sync::Mutex<()> = std::sync::Mutex::new(()); - let _guard = GENERATING.lock().unwrap_or_else(|e| e.into_inner()); let Some(ff) = ffmpeg() else { eprintln!("SKIP {}: ffmpeg not found", f.name); return None; }; let dir = fixtures_dir(); + // one generation at a time, across test processes too: tests share fixtures, and realtime + // libvpx encodes differ from run to run, so a stream and its reference decode must come from + // the same encode + // a lock failure is a broken environment, not a missing tool: fail rather than skip + let lock = std::fs::File::create(dir.join("vp8.generate.lock")).expect("create the fixture lock file"); + lock.lock().expect("lock the fixture lock file"); let ivf = dir.join(format!("{}.ivf", f.name)); let yuv = dir.join(format!("{}.yuv", f.name)); if !ivf.exists() { diff --git a/crates/vp8/tests/conformance.rs b/crates/vp8/tests/it/conformance.rs similarity index 99% rename from crates/vp8/tests/conformance.rs rename to crates/vp8/tests/it/conformance.rs index 27c7b92aa..3f120877f 100644 --- a/crates/vp8/tests/conformance.rs +++ b/crates/vp8/tests/it/conformance.rs @@ -1,7 +1,7 @@ //! Bit-exactness against libvpx (through ffmpeg) on generated fixtures covering every coding //! tool: skipped when ffmpeg or its libvpx encoder is absent. -mod common; +use crate::common; fn check(name: &str) { let f = common::fixture(name); diff --git a/crates/vp8/tests/it/main.rs b/crates/vp8/tests/it/main.rs new file mode 100644 index 000000000..ffc58b90c --- /dev/null +++ b/crates/vp8/tests/it/main.rs @@ -0,0 +1,4 @@ +mod common; +mod conformance; +mod robustness; +mod vectors; diff --git a/crates/vp8/tests/robustness.rs b/crates/vp8/tests/it/robustness.rs similarity index 99% rename from crates/vp8/tests/robustness.rs rename to crates/vp8/tests/it/robustness.rs index a4027f15d..0c5081402 100644 --- a/crates/vp8/tests/robustness.rs +++ b/crates/vp8/tests/it/robustness.rs @@ -1,6 +1,6 @@ //! Damaged, truncated, reordered and random input must never panic (errors are fine). -mod common; +use crate::common; use filmcraft_vp8::{Decoder, Error}; diff --git a/crates/vp8/tests/vectors.rs b/crates/vp8/tests/it/vectors.rs similarity index 99% rename from crates/vp8/tests/vectors.rs rename to crates/vp8/tests/it/vectors.rs index 789a40f56..d992c1926 100644 --- a/crates/vp8/tests/vectors.rs +++ b/crates/vp8/tests/it/vectors.rs @@ -3,7 +3,7 @@ //! downloaded by the tests; fetch them from //! `https://storage.googleapis.com/downloads.webmproject.org/test_data/libvpx/`. -mod common; +use crate::common; /// MD5 (RFC 1321), for comparing with the vectors' checksums. fn md5(data: &[u8]) -> String { diff --git a/crates/vp9/README.md b/crates/vp9/README.md index aea3a7781..765eb4b0c 100644 --- a/crates/vp9/README.md +++ b/crates/vp9/README.md @@ -115,16 +115,16 @@ draft mode. - Draft mode (`draft_mode_changes_only_flagged_non_reference_frames`): on a two-layer temporal scalability stream, single- and frame-threaded, every unflagged frame is bit-exact and only flagged frames differ. -- Conformance (`tests/conformance.rs`): the fixture matrix below plus a stream concatenating two +- Conformance (`tests/it/conformance.rs`): the fixture matrix below plus a stream concatenating two sizes. `compare_detects_mismatch` guards the harness. `coverage_report` (ignored) prints the decoder statistics for each fixture. -- Synthetic (`tests/synthetic.rs`): headers of libvpx output are rewritten to produce what ffmpeg's +- Synthetic (`tests/it/synthetic.rs`): headers of libvpx output are rewritten to produce what ffmpeg's libvpx wrapper cannot: inter frames of a stream encoded at another size spliced after a 352x288 key frame with explicit frame sizes, i.e. **scaled reference prediction** (downscale 256x208, upscale 480x400, odd 301x199; a same-size splice is the control); a key frame rewritten as a hidden **intra-only frame** (reset_frame_context 3 / 2 / 0, various refresh masks) shown with **show_existing_frame**; show_existing_frame of every slot, also inside a superframe. -- Robustness (`tests/robustness.rs`): bit flips, byte replacement and truncation of real streams, +- Robustness (`tests/it/robustness.rs`): bit flips, byte replacement and truncation of real streams, truncated frames, random data and oversized frames never panic (debug and release) or hang, single- and multi-threaded. @@ -158,7 +158,7 @@ Backward adaptation is on (`-frame-parallel 0`) unless noted. ## Performance -`cargo test --release -p filmcraft-vp9 --test perf -- --ignored --nocapture` (bit-exactness is +`cargo test --release -p filmcraft-vp9 --test it perf:: -- --ignored --nocapture` (bit-exactness is verified first), or `VP9_THREADS=n VP9_LOOPS=k cargo run --release -p filmcraft-vp9 --example vp9dec -- in.ivf` (decodes the file k times and prints the statistics; `VP9_DRAFT=1` for draft mode). diff --git a/crates/vp9/tests/fuzz_hostile.rs b/crates/vp9/tests/fuzz_hostile.rs index 70f884ec0..a04227331 100644 --- a/crates/vp9/tests/fuzz_hostile.rs +++ b/crates/vp9/tests/fuzz_hostile.rs @@ -90,14 +90,63 @@ fn run_all(frames: &[Vec], label: &str) { assert!(res.is_ok() && PANICS.load(Ordering::SeqCst) == before, "panic: {label}"); } -/// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt). -fn iters() -> usize { - std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()).unwrap_or(300) +const VALUES: [u8; 5] = [0, 0xff, 0x80, 0x01, 0x7f]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 200 truncation +/// points spread over each of the first three frames, every header byte, overwrite value and +/// bit flip) plus N random rounds per seed: 300 is the full depth, more hunts longer. Unset, CI +/// runs a third of the truncation points and of the full depth's random rounds and, for each +/// header byte of each frame, one value and one bit, shifted by the frame index. A rotation +/// offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC day number), so every +/// case of the full sweep runs within a few days, and a random round's mutations match the same +/// round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 200, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=300 runs everything)"); + Depth { full: false, truncations: 200, rounds: 300, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, fi: usize, p: usize, i: usize) -> bool { + self.full || i == (p + fi + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, fi: usize, p: usize, b: usize) -> bool { + self.full || b == (p + fi + self.rotation % 8) % 8 + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) } /// Seconds the whole sweep may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). const DEADLINE_SECS: u64 = 600; +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=300` runs all of it (see [`Depth`]). #[test] fn hostile_streams_never_panic() { filmcraft_testkit::with_deadline(DEADLINE_SECS, sweep); @@ -105,6 +154,7 @@ fn hostile_streams_never_panic() { fn sweep() { let Some(ff) = filmcraft_testkit::ffmpeg_or_skip("VP9 fuzz") else { return }; + let depth = Depth::from_env(); for &(name, args, pix) in SEEDS { if !filmcraft_testkit::ffmpeg_has_encoder(&ff, "libvpx-vp9") { filmcraft_testkit::skip("VP9 fuzz", &format!("this ffmpeg has no libvpx-vp9 encoder (seed {name})")); @@ -120,7 +170,7 @@ fn sweep() { // truncate each of the first frames at every length for fi in 0..seed.len().min(3) { let len = seed[fi].len(); - for n in (0..len).step_by((len / 200).max(1)) { + for n in depth.truncations(len) { let mut d = seed.clone(); d[fi].truncate(n); run_all(&d, &format!("{name} frame {fi} truncate {n}")); @@ -129,12 +179,18 @@ fn sweep() { // header region of every frame: every byte to a few values, every bit for fi in 0..seed.len() { for p in 0..seed[fi].len().min(96) { - for v in [0u8, 0xff, 0x80, 0x01, 0x7f] { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(fi, p, i) { + continue; + } let mut d = seed.clone(); d[fi][p] = v; run_all(&d, &format!("{name} frame {fi} byte {p}={v:#x}")); } for b in 0..8 { + if !depth.bit(fi, p, b) { + continue; + } let mut d = seed.clone(); d[fi][p] ^= 1 << b; run_all(&d, &format!("{name} frame {fi} flip {p}.{b}")); @@ -142,7 +198,8 @@ fn sweep() { } } let mut rng = Rng(0x9e3779b97f4a7c15 ^ name.len() as u64); - for i in 0..iters() { + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { let mut d = seed.clone(); for _ in 0..1 + rng.below(6) { let fi = rng.below(d.len()); @@ -156,7 +213,9 @@ fn sweep() { d[fi][p] = rng.next() as u8; } } - run_all(&d, &format!("{name} random {i}")); + if depth.round(i) { + run_all(&d, &format!("{name} random {i}")); + } } } } diff --git a/crates/vp9/tests/common/mod.rs b/crates/vp9/tests/it/common/mod.rs similarity index 100% rename from crates/vp9/tests/common/mod.rs rename to crates/vp9/tests/it/common/mod.rs diff --git a/crates/vp9/tests/conformance.rs b/crates/vp9/tests/it/conformance.rs similarity index 99% rename from crates/vp9/tests/conformance.rs rename to crates/vp9/tests/it/conformance.rs index ceb875e0b..58a7e21ad 100644 --- a/crates/vp9/tests/conformance.rs +++ b/crates/vp9/tests/it/conformance.rs @@ -1,7 +1,7 @@ //! Bit-exactness against ffmpeg's VP9 decoder on libvpx-vp9 fixtures (skipped when ffmpeg or //! libvpx is absent). -mod common; +use crate::common; macro_rules! fixture_tests { ($($name:ident),* $(,)?) => { diff --git a/crates/vp9/tests/it/main.rs b/crates/vp9/tests/it/main.rs new file mode 100644 index 000000000..c26f9adcf --- /dev/null +++ b/crates/vp9/tests/it/main.rs @@ -0,0 +1,5 @@ +mod common; +mod conformance; +mod perf; +mod robustness; +mod synthetic; diff --git a/crates/vp9/tests/perf.rs b/crates/vp9/tests/it/perf.rs similarity index 92% rename from crates/vp9/tests/perf.rs rename to crates/vp9/tests/it/perf.rs index 162392ba1..ad427ea1a 100644 --- a/crates/vp9/tests/perf.rs +++ b/crates/vp9/tests/it/perf.rs @@ -1,7 +1,7 @@ //! Decode speed (ignored by default; run with -//! `cargo test --release -p filmcraft-vp9 --test perf -- --ignored --nocapture`). +//! `cargo test --release -p filmcraft-vp9 --test it perf:: -- --ignored --nocapture`). -mod common; +use crate::common; use filmcraft_vp9::Decoder; use std::time::Instant; diff --git a/crates/vp9/tests/robustness.rs b/crates/vp9/tests/it/robustness.rs similarity index 99% rename from crates/vp9/tests/robustness.rs rename to crates/vp9/tests/it/robustness.rs index be63331d9..80db3d0fb 100644 --- a/crates/vp9/tests/robustness.rs +++ b/crates/vp9/tests/it/robustness.rs @@ -1,6 +1,6 @@ //! Damaged, truncated and random input must never panic or hang (errors are fine). -mod common; +use crate::common; use filmcraft_vp9::Decoder; use std::time::{Duration, Instant}; diff --git a/crates/vp9/tests/synthetic.rs b/crates/vp9/tests/it/synthetic.rs similarity index 96% rename from crates/vp9/tests/synthetic.rs rename to crates/vp9/tests/it/synthetic.rs index a01c7a1b8..64094f800 100644 --- a/crates/vp9/tests/synthetic.rs +++ b/crates/vp9/tests/it/synthetic.rs @@ -7,7 +7,7 @@ //! show_existing_frame; //! - show_existing_frame of hidden alt-ref frames and of arbitrary slots. -mod common; +use crate::common; use std::path::Path; @@ -236,10 +236,15 @@ fn encode(name: &str, w: u32, h: u32, frames: u32, extra: &[&str]) -> Option` writes `view-*.png`) | -| Settings | `crates/engine/src/settings_tests.rs`, `crates/ui-egui/tests/settings_ui.rs` | schema keys and defaults, persistence in the data directory, v1 → v2 migration and repair of bad values, validation, per-category reset; each wired setting (still / transition durations, step many, label defaults and colours, media scaling, timebase, recent projects, media cache policy, output mapping, smart quotes, auto-transcribe); the dialog: every category from its command and Cmd+,, edits by automation id, OK / Cancel / Escape / Reset…, theme, label names in Edit ▸ Label, tooltips, frame cache (`FILMCRAFT_UI_SHOTS=` writes `settings-*.png`) | -| Scopes and panels (M8.9 / M12.6) | `crates/scopes/src/tests.rs`, `crates/engine/src/{scopes,panels}_tests.rs`, `crates/ui-egui/tests/panels_ui.rs` | scope maths on generated frames (flat colours → exact histogram bins, ramps → waveform row = code value, 75 % bars → parade levels and vectorscope target cells for BT.601/709/2020, HLS hue angles, YC chroma, Clamp Signal, decimation, NaN); `scopes.read` on a colour matte (exact bins, 78.43 %) and on Bars and Tone (the six targets); metadata edits (one undo step, read-only fields refused, saved in the project), the event log (failed / disabled commands, repeats, jobs started / finished / failed / cancelled); every panel from Window ▸, the scopes' wrench menu, presets, five-scope grid, typing into a Metadata field, Timecode rows and modes, Events filter / Clear All, Progress cancel, Reference Monitor park / gang / scopes (`FILMCRAFT_UI_SNAPSHOT_DIR=` writes `panels-*.png`); `perf_scopes_at_1080p` (ignored) times each scope | -| Keyframe navigator and Effect Controls ruler | `crates/engine/src/tests.rs` (`add_keyframe_toggles_the_keyframe_at_the_playhead`), `crates/ui-egui/tests/keyframes_ui.rs`, `lane_tests` in `crates/ui-egui/src/panels/effect_controls.rs` | `effects.addKeyframe` adds a keyframe at the playhead or removes the one there (the value falls back to the remaining keyframes; undo, redo, parameters that name nothing); headless: the Properties diamond turns animation on, adds and removes the keyframe at the playhead (Scale 50 % → 100 % again), its ◀ ▶ arrows and Effect Controls' step between keyframes, Crop is applied before its first keyframe; a click on the Effect Controls time ruler and a drag of the playhead's handle move the playhead, and no playhead is drawn off the clip; ruler labels are sequence timecode, and clip times, frame rates and panel widths at their limits don't panic (`FILMCRAFT_UI_SHOTS=` writes `keyframes-*.png`) | +| Scripted UI | `crates/ui-egui/tests/it/scripted.rs` | headless app (egui_kittest) driven over the control channel: razor/undo, insert, apply effect, playback, click by automation id (§4) | +| Monitor view / Graphics menu | `crates/ui-egui/tests/it/view_ui.rs`, `crates/engine/src/graphics_tests.rs` | View menu items and checkmarks, playback/paused resolution, channel / comparison / multi-camera / waveform display modes, magnification and Hand-tool panning, guides dragged from the rulers (move, lock, remove, Add Guide…, templates in the preferences), snapping a graphic to the frame centre and a guide; align to frame / as group / to selection, distribute (centres and gaps), arrange, select next/previous graphic and layer, reset parameters / duration, vertical text, New Layer from file (`FILMCRAFT_UI_SHOTS=` writes `view-*.png`) | +| Settings | `crates/engine/src/settings_tests.rs`, `crates/ui-egui/tests/it/settings_ui.rs` | schema keys and defaults, persistence in the data directory, v1 → v2 migration and repair of bad values, validation, per-category reset; each wired setting (still / transition durations, step many, label defaults and colours, media scaling, timebase, recent projects, media cache policy, output mapping, smart quotes, auto-transcribe); the dialog: every category from its command and Cmd+,, edits by automation id, OK / Cancel / Escape / Reset…, theme, label names in Edit ▸ Label, tooltips, frame cache (`FILMCRAFT_UI_SHOTS=` writes `settings-*.png`) | +| Scopes and panels (M8.9 / M12.6) | `crates/scopes/src/tests.rs`, `crates/engine/src/{scopes,panels}_tests.rs`, `crates/ui-egui/tests/it/panels_ui.rs` | scope maths on generated frames (flat colours → exact histogram bins, ramps → waveform row = code value, 75 % bars → parade levels and vectorscope target cells for BT.601/709/2020, HLS hue angles, YC chroma, Clamp Signal, decimation, NaN); `scopes.read` on a colour matte (exact bins, 78.43 %) and on Bars and Tone (the six targets); metadata edits (one undo step, read-only fields refused, saved in the project), the event log (failed / disabled commands, repeats, jobs started / finished / failed / cancelled); every panel from Window ▸, the scopes' wrench menu, presets, five-scope grid, typing into a Metadata field, Timecode rows and modes, Events filter / Clear All, Progress cancel, Reference Monitor park / gang / scopes (`FILMCRAFT_UI_SNAPSHOT_DIR=` writes `panels-*.png`); `perf_scopes_at_1080p` (ignored) times each scope | +| Keyframe navigator and Effect Controls ruler | `crates/engine/src/tests.rs` (`add_keyframe_toggles_the_keyframe_at_the_playhead`), `crates/ui-egui/tests/it/keyframes_ui.rs`, `lane_tests` in `crates/ui-egui/src/panels/effect_controls.rs` | `effects.addKeyframe` adds a keyframe at the playhead or removes the one there (the value falls back to the remaining keyframes; undo, redo, parameters that name nothing); headless: the Properties diamond turns animation on, adds and removes the keyframe at the playhead (Scale 50 % → 100 % again), its ◀ ▶ arrows and Effect Controls' step between keyframes, Crop is applied before its first keyframe; a click on the Effect Controls time ruler and a drag of the playhead's handle move the playhead, and no playhead is drawn off the clip; ruler labels are sequence timecode, and clip times, frame rates and panel widths at their limits don't panic (`FILMCRAFT_UI_SHOTS=` writes `keyframes-*.png`) | | Golden images | `crates/golden/tests/golden.rs` | CPU renders vs committed PNGs; GPU vs CPU on the same scenes (§3) | | Engine / command | `crates/engine/src/tests.rs` | run commands on the demo project, assert the sequence, undo/redo, disabled cases | | Render | `crates/render/src/tests.rs` | compositing, opacity, Motion, cross dissolve midpoint, ½-res vs full, GPU plan vs reference, audio mix, audio-effect continuity | -| Multi-camera / sync | `crates/audio-dsp/src/sync.rs`, `crates/engine/src/multicam_tests.rs`, `crates/ui-egui/tests/multicam_ui.rs` | audio offsets within one sample: Rust-generated multi-mic speech (gains −20…+10 dB, SNR 40/25/10/3 dB, zero-phase colouring, fractional delays, a 0 dB-SNR room with a −4 dB echo) and ffmpeg-made camera/recorder files (pink-noise bursts, independent noise; exact lags recovered); timecode (± hours) and marker sync within a frame; Merge Clips; timeline Synchronize keeps frame-aligned starts; multicam structure, angle render (PSNR vs the camera), Switch Audio mix = the angle's track, live switching from a simulated playback clock (cuts, healing, one undo step), Ctrl+N cuts, flatten (frames identical), save/load; 20 angles: paging (auto / 2×2 / 3×3), page-relative keys, absolute cameras beyond 16, a page decoding only its angles at the cell scale (recording source), Auto-Adjust scale, Selection Top Down; headless: page arrows, angle 17–20 cells, grid-only view, Edit Cameras thumbnails; `perf_grid_of_four_1080p_angles` (ignored) prints the grid cost | -| Project panel and Media Browser (M12.7) | `crates/engine/src/{project_panel,media_browser}_tests.rs`, `crates/ui-egui/tests/project_panel_ui.rs` | view settings and their preference round trip; Metadata Display columns (custom fields, Freeform keys hidden), numeric vs text sorting, view presets (save / save as / restore / rename / delete, ten slots, persisted in `preferences.json`); Freeform moves keeping offsets, snapping, Align to Grid, Clip Size, stacks, arrangements, Reset (each one undo step, saved in the project); bin rename / nesting; the Media Browser against a fake filesystem (folders first, hidden and unsupported files left out, file-type filter, back / forward / up, recent directories, Favorites, import of a selection or a folder, Open in Source Monitor, ingest, probe columns). Headless: List view (sort by header, drag a column wider, inline rename), Icon view (hover scrub + I / O set the clip's In / Out, Shift+] / [), Freeform (drag a card, stack from the card menu, options dialog), panel menu, Metadata Display dialog, view presets, bins in place / tab / window, footer buttons, Media Browser navigation, Favorites, import, Shift+O, Edit Columns…, thumbnails, drag to the Timeline (`FILMCRAFT_UI_SNAPSHOT_DIR=` writes `project-*.png`, `media-browser-*.png`) | -| Menu long tail (M3.11) | `crates/render/src/scene.rs`, `crates/engine/src/{scene_detect,sequence_extras,project_tools}_tests.rs`, `crates/interchange/src/ale.rs`, `crates/project/src/find.rs`, `crates/ui-egui/tests/menus_ui.rs` | Scene Edit Detection on synthetic frames (hard cuts vs pans, flashes, sensitivity) and on generated movies with known cut points (our own ProRes render of demo scenes; an ffmpeg-made H.264 concat of testsrc2 / SMPTE bars / Mandelbrot): cuts with linked audio, clip markers, subclips, one undo step, background job and cancel; Normalize Mix Track to a target peak; Simplify Sequence; Find / Find Next and search bins; Automate to Sequence (overlap, transitions, insert, unnumbered markers); ALE round trip; Project Settings and scratch disks; menu order and shortcuts against Premiere's; the dialogs by automation id | +| Multi-camera / sync | `crates/audio-dsp/src/sync.rs`, `crates/engine/src/multicam_tests.rs`, `crates/ui-egui/tests/it/multicam_ui.rs` | audio offsets within one sample: Rust-generated multi-mic speech (gains −20…+10 dB, SNR 40/25/10/3 dB, zero-phase colouring, fractional delays, a 0 dB-SNR room with a −4 dB echo) and ffmpeg-made camera/recorder files (pink-noise bursts, independent noise; exact lags recovered); timecode (± hours) and marker sync within a frame; Merge Clips; timeline Synchronize keeps frame-aligned starts; multicam structure, angle render (PSNR vs the camera), Switch Audio mix = the angle's track, live switching from a simulated playback clock (cuts, healing, one undo step), Ctrl+N cuts, flatten (frames identical), save/load; 20 angles: paging (auto / 2×2 / 3×3), page-relative keys, absolute cameras beyond 16, a page decoding only its angles at the cell scale (recording source), Auto-Adjust scale, Selection Top Down; headless: page arrows, angle 17–20 cells, grid-only view, Edit Cameras thumbnails; `perf_grid_of_four_1080p_angles` (ignored) prints the grid cost | +| Project panel and Media Browser (M12.7) | `crates/engine/src/{project_panel,media_browser}_tests.rs`, `crates/ui-egui/tests/it/project_panel_ui.rs` | view settings and their preference round trip; Metadata Display columns (custom fields, Freeform keys hidden), numeric vs text sorting, view presets (save / save as / restore / rename / delete, ten slots, persisted in `preferences.json`); Freeform moves keeping offsets, snapping, Align to Grid, Clip Size, stacks, arrangements, Reset (each one undo step, saved in the project); bin rename / nesting; the Media Browser against a fake filesystem (folders first, hidden and unsupported files left out, file-type filter, back / forward / up, recent directories, Favorites, import of a selection or a folder, Open in Source Monitor, ingest, probe columns). Headless: List view (sort by header, drag a column wider, inline rename), Icon view (hover scrub + I / O set the clip's In / Out, Shift+] / [), Freeform (drag a card, stack from the card menu, options dialog), panel menu, Metadata Display dialog, view presets, bins in place / tab / window, footer buttons, Media Browser navigation, Favorites, import, Shift+O, Edit Columns…, thumbnails, drag to the Timeline (`FILMCRAFT_UI_SNAPSHOT_DIR=` writes `project-*.png`, `media-browser-*.png`) | +| Menu long tail (M3.11) | `crates/render/src/scene.rs`, `crates/engine/src/{scene_detect,sequence_extras,project_tools}_tests.rs`, `crates/interchange/src/ale.rs`, `crates/project/src/find.rs`, `crates/ui-egui/tests/it/menus_ui.rs` | Scene Edit Detection on synthetic frames (hard cuts vs pans, flashes, sensitivity) and on generated movies with known cut points (our own ProRes render of demo scenes; an ffmpeg-made H.264 concat of testsrc2 / SMPTE bars / Mandelbrot): cuts with linked audio, clip markers, subclips, one undo step, background job and cancel; Normalize Mix Track to a target peak; Simplify Sequence; Find / Find Next and search bins; Automate to Sequence (overlap, transitions, insert, unnumbered markers); ALE round trip; Project Settings and scratch disks; menu order and shortcuts against Premiere's; the dialogs by automation id | | Mixer | `crates/render/src/mixer_tests.rs`, `crates/engine/src/mixer_tests.rs` | sample-exact fader gain and pan laws, automation at block boundaries (bit-identical however requests are cut), solo/mute/solo-safe, sends and submix routing, latency-compensation alignment, render-vs-playback identity, Touch ramp-back, recorder modes (Latch/Touch/Write), thinning, Audio Gain, transition curves; `perf_24_tracks_3_effects_realtime_factor` prints the realtime factor | -| 5.1 / multichannel | `crates/audio-dsp/src/channels.rs`, `crates/render/src/mixer_tests.rs`, `crates/export/src/surround_tests.rs`, `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | BS.775 downmix / upmix coefficients exact, mixdown types; 5.1 panner point gains equal-power (Σg² = 1 over a 21×21 grid × 3 centre settings), speaker positions exact, 5.1 through the default puck bit-exact identity; graph: stereo/mono tracks into a 5.1 Mix (L/R, C, rear right, room centre −6 dB each, LFE = mean), 5.1 track through a 5.1 Mix and folded into stereo, a 5.1 submix, `pan51` automation block-invariant, six-channel meters; export: 6-ch WAV (`WAVE_FORMAT_EXTENSIBLE`, mask 0x3F) and ProRes MOV (6 × 24-bit PCM) decoded by ffmpeg **sample-exact**, ffprobe `channels` 6 / `channel_layout` 5.1; AAC 5.1 in MP4: every decoded channel's own tone ≥ 30 dB above the others' (no swapped channels), stereo export of a 5.1 Mix = BS.775 within 2·10⁻⁶; `sequence.settings {mix}`, `file.newSequence {mix, trackType}`, panner lanes, Write pass records the puck; UI: puck drag (one undo step), Center / LFE, six meter bars | -| Audio Clip Mixer automation, Track Mixer view | `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | Touch: a 201-point fader ramp thins to ≤ 6 clip keyframes, values within 0.1 dB, live override while held, values outside the gesture unchanged, one undo step; Latch holds until stop; Write records volume and pan from the start; a gesture across two clips writes both; Read commits once without a pass; UI: Clip Mixer mode dropdown, a recorded fader drag writes clip keyframes; Show/Hide Tracks, Meter Input(s) Only, transport record button | -| Voice-over | `crates/engine/src/voiceover_tests.rs`, `crates/ui-egui/tests/voiceover_ui.rs` | synthetic click train lands in `mix_sequence` exactly at R + k·period (pre-roll dropped, exact sample count), punch-in between In/Out (ramp source, every sample checked), one undo step + undo/redo, take numbering, record-arm/targeting track choice, sync/discard/zero-length/locked cases, cue times; UI: right-click Mic → Voice-Over Record Settings dialog (edits persist on OK, Cancel discards), the Mic button starts and stops a take on its track with pre-roll and overlay, and records past the end of the timeline (also `voiceover_playback_tests` in `crates/ui-egui/src/lib.rs`) | -| Text to Speech | `crates/tts/src/**/*.rs` (Kokoro tests need `FILMCRAFT_KOKORO_DIR`), `crates/tts-text`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties; natural voices: 50-sentence Whisper-base WER 2.1–3.6 % for all 9 voices (crate README), loaders fuzzed with hostile files, `tts.render` cache, download refused without the feature | -| Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id | -| Oracle | `crates/*/tests/*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) | -| Robustness / fuzz | `*/tests/robustness.rs`, `*/tests/fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic | -| Program monitor while dragging | `crates/ui-egui/tests/perf_ui.rs` | headless: a parameter changed on every UI pass (a value dragged in Effect Controls or Properties) and a backward scrub leave the Program monitor's picture at most one step behind (it shows the frame asked for the pass before while the frame due renders; it used to freeze until the mouse rested); a frame still rendering is kept when the same frame is asked for at a newer revision, so frames slower than a refresh (CPU effects) are shown too; `perf.stats` reports the picture's frame and revision against the playhead and the project | -| Performance | `*/tests/perf.rs` (`#[ignore]`) | §5 | +| 5.1 / multichannel | `crates/audio-dsp/src/channels.rs`, `crates/render/src/mixer_tests.rs`, `crates/export/src/surround_tests.rs`, `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/it/audio51_ui.rs` | BS.775 downmix / upmix coefficients exact, mixdown types; 5.1 panner point gains equal-power (Σg² = 1 over a 21×21 grid × 3 centre settings), speaker positions exact, 5.1 through the default puck bit-exact identity; graph: stereo/mono tracks into a 5.1 Mix (L/R, C, rear right, room centre −6 dB each, LFE = mean), 5.1 track through a 5.1 Mix and folded into stereo, a 5.1 submix, `pan51` automation block-invariant, six-channel meters; export: 6-ch WAV (`WAVE_FORMAT_EXTENSIBLE`, mask 0x3F) and ProRes MOV (6 × 24-bit PCM) decoded by ffmpeg **sample-exact**, ffprobe `channels` 6 / `channel_layout` 5.1; AAC 5.1 in MP4: every decoded channel's own tone ≥ 30 dB above the others' (no swapped channels), stereo export of a 5.1 Mix = BS.775 within 2·10⁻⁶; `sequence.settings {mix}`, `file.newSequence {mix, trackType}`, panner lanes, Write pass records the puck; UI: puck drag (one undo step), Center / LFE, six meter bars | +| Audio Clip Mixer automation, Track Mixer view | `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/it/audio51_ui.rs` | Touch: a 201-point fader ramp thins to ≤ 6 clip keyframes, values within 0.1 dB, live override while held, values outside the gesture unchanged, one undo step; Latch holds until stop; Write records volume and pan from the start; a gesture across two clips writes both; Read commits once without a pass; UI: Clip Mixer mode dropdown, a recorded fader drag writes clip keyframes; Show/Hide Tracks, Meter Input(s) Only, transport record button | +| Voice-over | `crates/engine/src/voiceover_tests.rs`, `crates/ui-egui/tests/it/voiceover_ui.rs` | synthetic click train lands in `mix_sequence` exactly at R + k·period (pre-roll dropped, exact sample count), punch-in between In/Out (ramp source, every sample checked), one undo step + undo/redo, take numbering, record-arm/targeting track choice, sync/discard/zero-length/locked cases, cue times; UI: right-click Mic → Voice-Over Record Settings dialog (edits persist on OK, Cancel discards), the Mic button starts and stops a take on its track with pre-roll and overlay, and records past the end of the timeline (also `voiceover_playback_tests` in `crates/ui-egui/src/lib.rs`) | +| Text to Speech | `crates/tts/src/**/*.rs` (Kokoro tests need `FILMCRAFT_KOKORO_DIR`), `crates/tts-text`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/it/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties; natural voices: 50-sentence Whisper-base WER 2.1–3.6 % for all 9 voices (crate README), loaders fuzzed with hostile files, `tts.render` cache, download refused without the feature | +| Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/it/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id | +| Oracle | `crates/*/tests/{,it/}*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) | +| Robustness / fuzz | `*/tests/{,it/}robustness.rs`, `*/tests/{,it/}fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic | +| Program monitor while dragging | `crates/ui-egui/tests/it/perf_ui.rs` | headless: a parameter changed on every UI pass (a value dragged in Effect Controls or Properties) and a backward scrub leave the Program monitor's picture at most one step behind (it shows the frame asked for the pass before while the frame due renders; it used to freeze until the mouse rested); a frame still rendering is kept when the same frame is asked for at a newer revision, so frames slower than a refresh (CPU effects) are shown too; `perf.stats` reports the picture's frame and revision against the playhead and the project | +| Performance | `*/tests/{,it/}perf.rs` (`#[ignore]`) | §5 | Commit `*.proptest-regressions` files so failing cases are re-run. @@ -52,7 +60,7 @@ are never linked or shipped ([AGENTS.md](../AGENTS.md) §2). - If a tool is missing the test prints `SKIPPED (): ffmpeg not found …` and passes. Set **`FILMCRAFT_REQUIRE_ORACLES=1`** (CI) to make every such skip a failure. In new tests use `let ff = filmcraft_testkit::require_ffmpeg!();` (also `require_ffprobe!`, `require_oracles!`). -- Some oracles need a recent ffmpeg: the APV tests (`crates/apv/tests/oracle.rs`) skip when +- Some oracles need a recent ffmpeg: the APV tests (`crates/apv/tests/it/oracle.rs`) skip when `ffmpeg -formats` does not list `apv` (ffmpeg ≥ 8.0), naming the detected version. - Fixtures are generated on first use into `/target/fixtures//` (`filmcraft_testkit::fixtures_dir`; independent of `CARGO_TARGET_DIR`, so agents with private @@ -71,12 +79,12 @@ are never linked or shipped ([AGENTS.md](../AGENTS.md) §2). the matrix. - Fixture sources are synthetic: `testsrc2`, `mandelbrot`, SMPTE bars, noise and fades, sine tones. H.264 and HEVC fixtures need ffmpeg built with libx264 and libx265. VideoToolbox fixtures are - generated only on macOS. The Media Foundation parity tests (`crates/platform/tests/media_foundation.rs`, + generated only on macOS. The Media Foundation parity tests (`crates/platform/tests/it/media_foundation.rs`, Windows only) make their 640x360, 1080p and 2160p H.264 / HEVC / Main 10 fixtures with ffmpeg and skip without a Direct3D 11 video device or the HEVC / VP9 / AV1 codec extensions of the Microsoft Store. The VP9 and AV1 fixtures (libvpx-vp9, libaom-av1; 360p, 1080p, 2160p, hidden alt-ref frames, two GOPs) need an ffmpeg with those encoders. -The NVENC tests (`crates/platform/tests/nvenc.rs` and `nvenc_export.rs` for H.264, `nvenc_rgba_input.rs` for +The NVENC tests (`crates/platform/tests/it/nvenc.rs` and `nvenc_export.rs` for H.264, `nvenc_rgba_input.rs` for the GPU's RGB → 4:2:0 conversion, `nvenc_hevc.rs`, `nvenc_hevc_export.rs`, `nvenc_hevc_probe.rs` and `nvenc_hevc_warm.rs` for H.265, `nvenc_hevc_main10.rs` and `nvenc_hevc_hdr_export.rs` for Main 10 HDR; Windows only) skip, printing `SKIPPED`, without an NVIDIA GPU @@ -173,7 +181,7 @@ Each codec README has the full fixture matrix and the measured results. ### Scripted UI tests (headless) -`crates/ui-egui/tests/scripted.rs` runs the real `FilmcraftApp` under +`crates/ui-egui/tests/it/scripted.rs` runs the real `FilmcraftApp` under [`egui_kittest`](https://docs.rs/egui_kittest) (`build_eframe`): no window, no GPU, no OS event loop. A small `Driver` opens the demo project, sends requests through the same control channel agents use (`ControlRequest` → `control::handle`, exactly as the TCP server does) and steps egui frames until @@ -190,15 +198,15 @@ next frame by the app's own `raw_input_hook`, so clicks by automation id work he | `clicking_a_tool_button_by_automation_id` | `ui.click {id: "tools.Razor"}` changes the tool (real egui input path) | | `unknown_methods_and_commands_fail_cleanly` | errors come back as `{"ok": false}` | -`crates/ui-egui/tests/mixer_ui.rs` drives the Audio Track Mixer, Audio Clip Mixer, timeline track +`crates/ui-egui/tests/it/mixer_ui.rs` drives the Audio Track Mixer, Audio Clip Mixer, timeline track keyframes and the Audio Gain dialog by automation id and with multi-frame pointer drags (press, move over several frames, release). With `FILMCRAFT_UI_SNAPSHOT_DIR=` it also renders the window offscreen through wgpu (`Harness::render`) and writes `mixer-*.png`; this works without a visible window (for example on a locked screen, where `ui.screenshot` cannot capture). -`crates/ui-egui/tests/essential_sound_ui.rs` does the same for the Essential Sound panel (type buttons, +`crates/ui-egui/tests/it/essential_sound_ui.rs` does the same for the Essential Sound panel (type buttons, switches, a slider drag as one undo step, section bypass, Auto-Match, ducking, Browse presets; `essential-sound-*.png`). -`crates/ui-egui/tests/clip_audio_ui.rs` covers the audio of video clips (#223): the linked audio's +`crates/ui-egui/tests/it/clip_audio_ui.rs` covers the audio of video clips (#223): the linked audio's Volume, Channel Volume and Panner in Effect Controls and Properties, and the Volume line on audio clips in the Timeline (a drag is one undo step, Pen-tool keyframes, keyframe drags, a click or right-click on the line still reaches the clip; `FILMCRAFT_UI_SHOTS=` writes `volume-*.png`). @@ -214,12 +222,12 @@ and the WAV export equals it. `perf_full_dialogue_chain_realtime_factor` (ignored; run with `--release`) prints the realtime factor of all nine Dialogue effects on one clip (22× on one core). -`crates/ui-egui/tests/export_ui.rs` drives Export mode: the settings column and Summary, editing a +`crates/ui-egui/tests/it/export_ui.rs` drives Export mode: the settings column and Summary, editing a setting turning the preset into Custom, the Preset Manager (search, favourite star, save, apply, delete), the queue panel (Send to Queue, Up/Down, Cancel, Start, Retry, Clear) with real encodes, the Export button and the header's Quick Export popup (`export-*.png` snapshots). -`crates/ui-egui/tests/color_ui.rs` drives colour features the same way (`color-*.png` snapshots): +`crates/ui-egui/tests/it/color_ui.rs` drives colour features the same way (`color-*.png` snapshots): Lumetri Input/Look LUT menus and section switches, the Interpret Footage ▸ Color Management and Sequence Color Management dialogs, and the HDR scopes of a PQ sequence. @@ -258,12 +266,12 @@ engine behaviour without a window. | Benchmark | Command | |---|---| -| H.264 decode, 1080p, threads sweep | `cargo test --release -p filmcraft-h264 --test perf -- --ignored --nocapture` | -| HEVC decode, 1080p / 2160p | `cargo test --release -p filmcraft-hevc --test perf -- --ignored --nocapture` | -| ProRes decode/encode | `cargo test --release -p filmcraft-prores --test perf -- --ignored --nocapture` | +| H.264 decode, 1080p, threads sweep | `cargo test --release -p filmcraft-h264 --test it perf:: -- --ignored --nocapture` | +| HEVC decode, 1080p / 2160p | `cargo test --release -p filmcraft-hevc --test it perf:: -- --ignored --nocapture` | +| ProRes decode/encode | `cargo test --release -p filmcraft-prores --test it perf:: -- --ignored --nocapture` | | Encoder speed / PSNR / bitrate | `cargo run --release -p filmcraft-h264enc --example h264enc_synth -- …` | | Decode a real file through the media stack | `cargo run --release -p filmcraft-cli -- bench-decode file.mp4 --frames 120` | -| Coding-tool coverage of the fixtures | `cargo test --release -p filmcraft-h264 --test conformance coverage_report -- --ignored --nocapture` (same for `hevc`) | +| Coding-tool coverage of the fixtures | `cargo test --release -p filmcraft-h264 --test it conformance::coverage_report -- --ignored --nocapture` (same for `hevc`) | | **Whole-app suite**: decode fps per codec, playback, scrubbing, timeline UI, export, project save/open, peak RSS | `cargo xtask bench` ([performance.md](performance.md)) | `cargo xtask bench` (the example `crates/ui-egui/examples/bench/`) runs six sections, each in its diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 819326fcf..1e7c2dc7f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -16,7 +16,8 @@ //! lists, against a reference list (default: the local `plan/premiere/.json`), presence and //! placement reported separately (`parity.rs`, docs/gaps.md G7). //! - `ico …`: pack PNGs into a Windows `.ico` (used by `packaging/icons.sh`). -//! - `ci`: fmt check, clippy -D warnings, tests, layers, assets, wasm. +//! - `ci`: fmt check, clippy -D warnings, tests (`cargo nextest run` when installed) and doctests, +//! layers, assets, wasm. mod ico; mod parity; @@ -259,7 +260,7 @@ const WEB_CRATES: &[&str] = &["filmcraft-ui-egui", "filmcraft-web"]; fn wasm() -> Result<(), String> { let md = metadata()?; let mut cmd = Command::new(env!("CARGO")); - cmd.args(["check", "--target", "wasm32-unknown-unknown"]); + cmd.args(["check", "--target", "wasm32-unknown-unknown", "--keep-going"]); let mut n = 0; for (name, _) in workspace_crates(&md) { if layer_of(&name).is_some_and(|l| l <= 4) || WEB_CRATES.contains(&name.as_str()) { @@ -400,26 +401,26 @@ fn serve_dir(dir: &std::path::Path, port: u16) -> Result<(), String> { Ok(()) } -/// (crate, test target) whose ignored `generate_fixtures` test builds that crate's fixture matrix. -const FIXTURE_GENERATORS: &[(&str, &str)] = &[ - ("h264", "conformance"), - ("hevc", "conformance"), - ("isobmff", "oracle_demux"), - ("matroska", "oracle"), - ("prores", "oracle_decode"), - ("dnx", "oracle_decode"), - ("codecs", "mxf_oracle"), - ("codecs", "mpeg_oracle"), - ("mpeg2v", "oracle"), - ("ac3", "oracle"), - ("mpa", "oracle"), - ("codecs", "ogg_oracle"), +/// (crate, test target, test) whose ignored test builds that crate's fixture matrix. +const FIXTURE_GENERATORS: &[(&str, &str, &str)] = &[ + ("h264", "it", "conformance::generate_fixtures"), + ("hevc", "it", "conformance::generate_fixtures"), + ("isobmff", "it", "oracle_demux::generate_fixtures"), + ("matroska", "oracle", "generate_fixtures"), + ("prores", "it", "oracle_decode::generate_fixtures"), + ("dnx", "it", "oracle_decode::generate_fixtures"), + ("codecs", "it", "mxf_oracle::generate_fixtures"), + ("codecs", "it", "mpeg_oracle::generate_fixtures"), + ("mpeg2v", "it", "oracle::generate_fixtures"), + ("ac3", "oracle", "generate_fixtures"), + ("mpa", "oracle", "generate_fixtures"), + ("codecs", "it", "ogg_oracle::generate_fixtures"), ]; fn fixtures(only: &[String]) -> Result<(), String> { - let unknown: Vec<&String> = only.iter().filter(|o| !FIXTURE_GENERATORS.iter().any(|(c, _)| *c == short(o))).collect(); + let unknown: Vec<&String> = only.iter().filter(|o| !FIXTURE_GENERATORS.iter().any(|(c, _, _)| *c == short(o))).collect(); if !unknown.is_empty() { - let known: Vec<&str> = FIXTURE_GENERATORS.iter().map(|(c, _)| *c).collect(); + let known: Vec<&str> = FIXTURE_GENERATORS.iter().map(|(c, _, _)| *c).collect(); return Err(format!("no fixture generator for {unknown:?} (known: {})", known.join(", "))); } let Some(ff) = filmcraft_testkit::ffmpeg() else { @@ -429,11 +430,11 @@ fn fixtures(only: &[String]) -> Result<(), String> { println!("fixtures: {}", filmcraft_testkit::fixtures::fixtures_root().display()); let (mut made, mut cached, mut skipped) = (0, 0, 0); let mut failed = Vec::new(); - for (krate, target) in FIXTURE_GENERATORS.iter().filter(|(c, _)| only.is_empty() || only.iter().any(|o| short(o) == *c)) { + for (krate, target, test) in FIXTURE_GENERATORS.iter().filter(|(c, _, _)| only.is_empty() || only.iter().any(|o| short(o) == *c)) { let pkg = format!("filmcraft-{krate}"); - println!("\n== {pkg} ({target})"); + println!("\n== {pkg} ({test})"); let mut cmd = Command::new(env!("CARGO")); - cmd.args(["test", "--release", "-p", &pkg, "--test", target, "--", "--ignored", "--exact", "generate_fixtures", "--nocapture"]); + cmd.args(["test", "--release", "-p", &pkg, "--test", target, "--", "--ignored", "--exact", test, "--nocapture"]); cmd.stderr(std::process::Stdio::inherit()); let out = cmd.output().map_err(|e| e.to_string())?; for line in String::from_utf8_lossy(&out.stdout).lines() { @@ -456,13 +457,28 @@ fn fixtures(only: &[String]) -> Result<(), String> { fn ci() -> Result<(), String> { let cargo = env!("CARGO"); run(Command::new(cargo).args(["fmt", "--check"]))?; - run(Command::new(cargo).args(["clippy", "--workspace", "--all-targets", "--release", "--", "-D", "warnings"]))?; - run(Command::new(cargo).args(["test", "--workspace", "--release"]))?; + run(Command::new(cargo).args(["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"]))?; + tests()?; layers()?; assets()?; wasm() } +/// Lib, bin and integration tests (examples aren't linked; clippy checked them), then doctests, +/// which nextest doesn't run. +fn tests() -> Result<(), String> { + let cargo = env!("CARGO"); + let targets = ["--workspace", "--lib", "--bins", "--tests", "--no-fail-fast"]; + let nextest = Command::new(cargo).args(["nextest", "--version"]).output().is_ok_and(|o| o.status.success()); + if nextest { + run(Command::new(cargo).args(["nextest", "run"]).args(targets))?; + } else { + println!("cargo-nextest not found, running cargo test (install: cargo install cargo-nextest --locked)"); + run(Command::new(cargo).arg("test").args(targets))?; + } + run(Command::new(cargo).args(["test", "--workspace", "--doc"])) +} + /// The headless playback benchmark (`crates/ui-egui/examples/bench_playback.rs`); extra /// arguments go to the benchmark (`cargo xtask bench-playback --scenario stack3 --res full`). fn bench_playback() -> Result<(), String> {