From ff8728af089dd6e21f3a526e752482f8bce6ebcc Mon Sep 17 00:00:00 2001 From: lloyd Date: Fri, 9 Oct 2026 18:28:56 +0200 Subject: [PATCH] Anamorphic media: pixel aspect ratio in placement, monitors and New Sequence From Clip A 1440x1080 clip with 4:3 pixels showed squeezed: the demuxers read its pixel aspect ratio, but nothing after import used it. New Sequence From Clip made a 1440x1080 sequence with square pixels, the Motion transform sized clips from storage pixels (pillarboxed at 1440 wide in a 1920x1080 sequence), and the monitors drew the frame at its storage aspect. - project: checked_par / sane_par / par_ratio / conformed_size treat file and project ratios as hostile (a zero term, or more than 8:1 either way, is square). MediaClip::pixel_aspect (Interpret Footage wins over the file), Project::source_par (subclips follow the parent, nested sequences have their own, graphics and adjustment layers have none) and Project::conformed_source_size. - render: motion_matrix takes the source's ratio and works in square display units (anchor in source pixels, position in sequence pixels, scale and rotation at the display aspect); Scale to Frame Size fits the display size. CPU, GPU plan, adjustment regions and the mask / graphics overlays share it. - engine: New Sequence From Clip, New Sequence from an item and multicam sequences copy the clip's ratio; default media scaling and Fit / Fill frame use the display size; generators made for a non-square sequence get its pixels; transcodes render pixel for pixel. clip.interpretFootage takes pixelAspect ([num, den] or "file"). - ui: the Source and Program monitors draw the picture at its display aspect; overlays map both axes; Sequence Settings shows the display aspect (16:9 for 1440x1080 at 4:3). Tests: render par_tests (fill, GPU plan parity, Scale to Frame Size, non-square sequences, rotation, Interpret Footage, hostile ratios), engine par_tests (New Sequence From Clip, overwrite into 1920x1080, default scaling, Fit / Fill, Interpret Footage with undo and refused values, hostile file ratios, generators, and ffmpeg-made MP4 and MKV files), project unit tests, and the headless par_ui monitor test. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/clip_ops.rs | 17 +- crates/engine/src/color.rs | 46 ++- crates/engine/src/commands.rs | 25 +- crates/engine/src/lib.rs | 2 + crates/engine/src/multicam.rs | 8 +- crates/engine/src/par_tests.rs | 231 +++++++++++++++ crates/engine/src/proxies.rs | 2 + crates/engine/src/settings.rs | 13 +- crates/project/src/lib.rs | 137 +++++++++ crates/render/src/lib.rs | 42 ++- crates/render/src/par_tests.rs | 270 ++++++++++++++++++ crates/render/src/plan.rs | 8 +- crates/ui-egui/src/menus.rs | 2 +- crates/ui-egui/src/panels/graphics.rs | 17 +- crates/ui-egui/src/panels/masks.rs | 2 +- crates/ui-egui/src/panels/monitor.rs | 12 +- crates/ui-egui/src/panels/monitor_view.rs | 11 + .../ui-egui/src/panels/sequence_settings.rs | 16 +- crates/ui-egui/tests/par_ui.rs | 111 +++++++ docs/architecture.md | 10 + docs/control-protocol.md | 2 +- 21 files changed, 933 insertions(+), 51 deletions(-) create mode 100644 crates/engine/src/par_tests.rs create mode 100644 crates/render/src/par_tests.rs create mode 100644 crates/ui-egui/tests/par_ui.rs diff --git a/crates/engine/src/clip_ops.rs b/crates/engine/src/clip_ops.rs index 1946e5cb7..cf0e22da2 100644 --- a/crates/engine/src/clip_ops.rs +++ b/crates/engine/src/clip_ops.rs @@ -639,7 +639,7 @@ fn sequence_from_clip(s: &mut Session, p: &Value) -> Result { let pi = s.project.item(first).ok_or_else(|| bad("file.newSequenceFromClip", "no such item"))?.clone(); let mut settings = match &pi.kind { ItemKind::Sequence(q) => q.settings.clone(), - _ => media_root(&s.project, first).map(|(_, m, _)| crate::commands::default_seq_settings_for(&m.info)).unwrap_or_default(), + _ => media_root(&s.project, first).map(|(_, m, _)| crate::commands::default_seq_settings_for(m)).unwrap_or_default(), }; if let ItemKind::AdjustmentLayer { width, height, rate, .. } = &pi.kind { settings.width = *width; @@ -1512,17 +1512,24 @@ fn fit_fill(s: &mut Session, p: &Value, fill: bool) -> Result { let clips = video_clips(s, p); let q = s.active_sequence().ok_or(EngineError::NoSequence)?; let (fw, fh) = (q.settings.width as f64, q.settings.height as f64); - let sizes: Vec<(ClipId, (u32, u32))> = - clips.iter().filter_map(|c| q.find_item(*c).and_then(|(_, i)| filmcraft_render::source_size(&s.project, i.item)).map(|sz| (*c, sz))).collect(); + let frame_par = q.settings.par; + // (clip, storage size for the anchor, size at the display aspect in sequence pixels for the scale) + let sizes: Vec<(ClipId, (u32, u32), (f64, f64))> = clips + .iter() + .filter_map(|c| { + let (_, i) = q.find_item(*c)?; + Some((*c, filmcraft_render::source_size(&s.project, i.item)?, s.project.conformed_source_size(i.item, frame_par)?)) + }) + .collect(); if sizes.is_empty() { return Err(bad(if fill { "clip.fillFrame" } else { "clip.fitToFrame" }, "select a video clip")); } let label = if fill { "Fill Frame" } else { "Fit to Frame" }; let out = s.edit_sequence(label, |q, _, _| { let mut out = Vec::new(); - for (c, (w, h)) in &sizes { + for (c, (w, h), (dw, dh)) in &sizes { let Some((_, it)) = q.find_item_mut(*c) else { continue }; - let (sx, sy) = (fw / (*w).max(1) as f64, fh / (*h).max(1) as f64); + let (sx, sy) = (fw / dw.max(1.0), fh / dh.max(1.0)); let pct = (if fill { sx.max(sy) } else { sx.min(sy) }) * 100.0; it.scale_to_frame = false; let Some(m) = it.effect_mut("motion") else { continue }; diff --git a/crates/engine/src/color.rs b/crates/engine/src/color.rs index 56049a7ee..71261bca5 100644 --- a/crates/engine/src/color.rs +++ b/crates/engine/src/color.rs @@ -40,7 +40,7 @@ pub(crate) fn commands() -> Vec { "clip.interpretFootage", "Interpret Footage…", &["Clip", "Modify"], - r#"{"items":[id]?,"colorSpace":"auto"|""}"#, + r#"{"items":[id]?,"colorSpace":"auto"|""?,"pixelAspect":[num,den]|"file"?}"#, has_footage, interpret, ), @@ -292,25 +292,57 @@ pub(crate) fn footage_targets(s: &Session, p: &Value) -> Vec { v } +/// `clip.interpretFootage`: the colour space (`colorSpace`) and / or the pixel aspect ratio +/// (`pixelAspect`) the footage is interpreted with. `"auto"` / `"file"` go back to the file's +/// metadata; a parameter left out keeps its current interpretation. fn interpret(s: &mut Session, p: &Value) -> Result { + const CMD: &str = "clip.interpretFootage"; let items = footage_targets(s, p); if items.is_empty() { - return Err(bad("clip.interpretFootage", "no footage selected")); + return Err(bad(CMD, "no footage selected")); } let cs = match str_p(p, "colorSpace") { - None => return Err(bad("clip.interpretFootage", "need `colorSpace` (auto or a colour space id)")), - Some("auto") | Some("") => None, - Some(c) => Some(ColorSpace::parse(c).ok_or_else(|| bad("clip.interpretFootage", format!("unknown colour space `{c}`")))?), + None => None, + Some("auto") | Some("") => Some(None), + Some(c) => Some(Some(ColorSpace::parse(c).ok_or_else(|| bad(CMD, format!("unknown colour space `{c}`")))?)), }; + let par = + match p.get("pixelAspect") { + None | Some(Value::Null) => None, + Some(Value::String(f)) if f == "file" || f == "auto" => Some(None), + Some(v) => Some(Some(pixel_aspect_p(v).ok_or_else(|| { + bad(CMD, format!("pixelAspect must be [num, den] between 1:{0} and {0}:1, or \"file\"", filmcraft_project::MAX_PIXEL_ASPECT)) + })?)), + }; + if cs.is_none() && par.is_none() { + return Err(bad(CMD, "need `colorSpace` (auto or a colour space id) or `pixelAspect`")); + } s.edit("Interpret Footage", |pr, _| { for i in &items { if let Some(ItemKind::Media(m)) = pr.item_mut(*i).map(|it| &mut it.kind) { - m.interpret.color_space = cs; + if let Some(cs) = cs { + m.interpret.color_space = cs; + } + if let Some(par) = par { + m.interpret.par = par; + } } } Ok(()) })?; - Ok(json!({"items": items.iter().map(|i| i.0).collect::>(), "colorSpace": cs.map(|c| c.id()).unwrap_or("auto")})) + let first = items.first().and_then(|i| s.project.item(*i)).and_then(|i| i.as_media()); + Ok(json!({ + "items": items.iter().map(|i| i.0).collect::>(), + "colorSpace": first.and_then(|m| m.interpret.color_space).map(|c| c.id()).unwrap_or("auto"), + "pixelAspect": first.map(|m| m.pixel_aspect()), + })) +} + +/// `[num, den]` as a pixel aspect ratio FilmCraft can use ([`filmcraft_project::checked_par`]). +fn pixel_aspect_p(v: &Value) -> Option<(u32, u32)> { + let [n, d] = v.as_array()?.as_slice() else { return None }; + let (n, d) = (u32::try_from(n.as_u64()?).ok()?, u32::try_from(d.as_u64()?).ok()?); + filmcraft_project::checked_par((n, d)) } fn spaces(_: &mut Session, _: &Value) -> Result { diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index cd02fab2d..f18e41ee6 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -357,12 +357,17 @@ pub fn with_links(s: &Session, clips: &[ClipId]) -> Vec { out } -pub(crate) fn default_seq_settings_for(info: &filmcraft_media::MediaInfo) -> SequenceSettings { +/// Sequence settings matching a media clip (New Sequence From Clip, New Sequence from an item): +/// its frame size, rate and pixel aspect ratio (Interpret Footage's, else the file's), so a +/// 1440 x 1080 clip with 4:3 pixels makes a 1440 x 1080 sequence that displays 16:9. +pub(crate) fn default_seq_settings_for(media: &MediaClip) -> SequenceSettings { + let info = &media.info; let mut st = SequenceSettings::default(); if let Some(v) = &info.video { st.width = v.width; st.height = v.height; st.frame_rate = v.frame_rate; + st.par = media.pixel_aspect(); st.preset = format!("{}x{} {}", v.width, v.height, v.frame_rate.label()); } if let Some(a) = &info.audio { @@ -455,8 +460,15 @@ fn new_generator(s: &mut Session, g: Generator, name: &str, label: Label, p: &Va let name = str_p(p, "name").unwrap_or(name).to_string(); let src = GeneratorSource::new(g, w, h, st.frame_rate, Tick::from_seconds_f64(secs)); let pool = s.media.clone(); + let par = filmcraft_project::sane_par(st.par); let id = s.edit(&format!("New {name}"), |pr, st| { let id = crate::demo::add_generator(pr, &pool, src, &name, label, None); + // made for the sequence: its pixels have the sequence's aspect, as in Premiere + if let Some(ItemKind::Media(m)) = pr.item_mut(id).map(|i| &mut i.kind) + && let Some(v) = m.info.video.as_mut() + { + v.par = par; + } st.project_selection = vec![id]; Ok(id) })?; @@ -521,6 +533,8 @@ pub(crate) fn place_item( _ => None, } .unwrap_or((1920, 1080)); + // `None`: drawn in the pixels of the sequence it is placed in + let src_par = s.project.source_par(item); let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; let media = s.media.clone(); let scaling = s.prefs.media.default_media_scaling.clone(); @@ -529,6 +543,7 @@ pub(crate) fn place_item( let seq = p.sequence(seq_id).ok_or(EngineError::NoSequence)?; let rate = seq.settings.frame_rate; let frame = (seq.settings.width, seq.settings.height); + let seq_par = seq.settings.par; let mut placements = Vec::new(); let link = if has_v && has_a { Some(p.alloc_id()) } else { None }; if let Some(vdest) = vdest.filter(|_| has_v) { @@ -537,8 +552,10 @@ pub(crate) fn place_item( for e in &mut v.effects { resolve_auto_points(e, frame, src_size); } - if is_media && src_size != frame { - crate::settings::apply_media_scaling(&mut v, &scaling, frame, src_size); + // the picture's size at its display aspect, in sequence pixels + let shown = filmcraft_project::conformed_size(src_size, src_par.unwrap_or(seq_par), seq_par); + if is_media && shown != (f64::from(frame.0), f64::from(frame.1)) { + crate::settings::apply_media_scaling(&mut v, &scaling, frame, shown); } placements.push((vdest, v)); } @@ -677,7 +694,7 @@ fn build() -> Vec { |s, p| { let mut st = SequenceSettings::default(); if let Some(from) = item_p(p, "fromItem").and_then(|i| s.project.item(i)).and_then(|i| i.as_media()) { - st = default_seq_settings_for(&from.info); + st = default_seq_settings_for(from); } if let Some(w) = checked_u32_p(p, "width", "file.newSequence")? { st.width = w; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3720264cf..b88121502 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1063,6 +1063,8 @@ mod nesting_tests; #[cfg(test)] mod panels_tests; #[cfg(test)] +mod par_tests; +#[cfg(test)] mod presets_tests; #[cfg(test)] mod previews_tests; diff --git a/crates/engine/src/multicam.rs b/crates/engine/src/multicam.rs index e5027ca08..23824942b 100644 --- a/crates/engine/src/multicam.rs +++ b/crates/engine/src/multicam.rs @@ -475,12 +475,12 @@ fn sync_items(s: &Session, items: &[ItemId], cmd: &str) -> Result> } fn settings_for(s: &Session, item: ItemId) -> SequenceSettings { - let info = match s.project.item(item).map(|i| &i.kind) { - Some(ItemKind::Media(m)) => Some(m.info.clone()), - Some(ItemKind::Subclip { parent, .. }) => s.project.item(*parent).and_then(|i| i.as_media()).map(|m| m.info.clone()), + let media = match s.project.item(item).map(|i| &i.kind) { + Some(ItemKind::Media(m)) => Some(m), + Some(ItemKind::Subclip { parent, .. }) => s.project.item(*parent).and_then(|i| i.as_media()), _ => None, }; - let mut st = info.as_ref().map(crate::commands::default_seq_settings_for).unwrap_or_default(); + let mut st = media.map(crate::commands::default_seq_settings_for).unwrap_or_default(); if let Some(m) = s.project.item(item).and_then(|i| i.as_media()) && let Some(r) = m.interpret.frame_rate { diff --git a/crates/engine/src/par_tests.rs b/crates/engine/src/par_tests.rs new file mode 100644 index 000000000..096fbf220 --- /dev/null +++ b/crates/engine/src/par_tests.rs @@ -0,0 +1,231 @@ +//! Anamorphic media: the pixel aspect ratio reaches New Sequence From Clip, the placement of clips +//! (default scaling, Fit / Fill) and the rendered frame, Interpret Footage overrides it, and +//! hostile ratios are square. Synthetic media (a Color Matte given 4:3 pixels) runs everywhere; +//! ffmpeg-made 1440 x 1080-style files (at a tenth of the size) check the MP4 and Matroska +//! demuxers end to end when ffmpeg is installed. + +use std::path::PathBuf; +use std::sync::Arc; + +use serde_json::json; + +use crate::Session; +use filmcraft_project::{ItemId, ItemKind, ParamValue}; + +/// A 144 x 108 Color Matte (a 1440 x 1080 stand-in) whose pixels are `par`. +fn matte(s: &mut Session, par: (u32, u32)) -> ItemId { + let r = s.execute("file.newColorMatte", json!({"color": "#ff0000", "width": 144, "height": 108, "seconds": 2.0})).unwrap(); + let id = ItemId(r["item"].as_u64().unwrap()); + if let Some(ItemKind::Media(m)) = Arc::make_mut(&mut s.project).item_mut(id).map(|i| &mut i.kind) { + m.info.video.as_mut().unwrap().par = par; + } + id +} + +/// The alpha of the middle row of the program frame at columns `xs`. +fn coverage(s: &Session, xs: &[usize]) -> Vec { + let img = s.render_program(1.0).unwrap(); + xs.iter().map(|&x| img.get(x, img.h / 2)[3]).collect() +} + +/// A 192 x 108 square-pixel sequence (a 1920 x 1080 stand-in) with `item` overwritten at 0. +fn overwrite_into_square_hd(s: &mut Session, item: ItemId) -> filmcraft_project::ClipId { + s.execute("file.newSequence", json!({"name": "HD", "width": 192, "height": 108, "fps": 24})).unwrap(); + s.execute("source.open", json!({"item": item.0})).unwrap(); + s.execute("playhead.set", json!({"seconds": 0.0})).unwrap(); + let r = s.execute("source.overwrite", json!({})).unwrap(); + filmcraft_project::ClipId(r["clips"][0].as_u64().unwrap()) +} + +fn motion_scale(s: &Session, clip: filmcraft_project::ClipId) -> f64 { + let (_, it) = s.active_sequence().unwrap().find_item(clip).unwrap(); + match it.effect("motion").unwrap().params["scale"].value { + ParamValue::Float(v) => v, + ref v => panic!("{v:?}"), + } +} + +#[test] +fn new_sequence_from_clip_carries_the_pixel_aspect() { + let mut s = Session::default(); + let item = matte(&mut s, (4, 3)); + let r = s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + let st = s.project.sequence(ItemId(r["sequence"].as_u64().unwrap())).unwrap().settings.clone(); + assert_eq!((st.width, st.height, st.par), (144, 108, (4, 3))); + assert!(st.validate().is_ok()); + // the clip maps pixel for pixel onto its own sequence: the whole frame + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + assert!(coverage(&s, &[0, 1, 72, 142, 143]).iter().all(|a| *a > 0.99)); + // New Sequence from the item too + s.execute("file.newSequence", json!({"name": "From item", "fromItem": item.0})).unwrap(); + assert_eq!(s.active_sequence().unwrap().settings.par, (4, 3)); + // a ratio written in other terms is reduced + let other = matte(&mut s, (16, 12)); + s.execute("file.newSequenceFromClip", json!({"items": [other.0]})).unwrap(); + assert_eq!(s.active_sequence().unwrap().settings.par, (4, 3)); +} + +#[test] +fn an_anamorphic_clip_fills_a_square_pixel_sequence() { + let mut s = Session::default(); + let item = matte(&mut s, (4, 3)); + let clip = overwrite_into_square_hd(&mut s, item); + // displayed 192 wide at 100 %: the whole frame (it was pillarboxed at 144 wide) + assert_eq!(motion_scale(&s, clip), 100.0); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + assert!(coverage(&s, &[1, 10, 96, 181, 190]).iter().all(|a| *a > 0.99), "{:?}", coverage(&s, &[1, 10, 96, 181, 190])); + // Fit / Fill fit the display size: already the frame size + s.execute("clip.fitToFrame", json!({"clips": [clip.0]})).unwrap(); + assert!((motion_scale(&s, clip) - 100.0).abs() < 1e-9); + s.execute("clip.fillFrame", json!({"clips": [clip.0]})).unwrap(); + assert!((motion_scale(&s, clip) - 100.0).abs() < 1e-9); + assert!(coverage(&s, &[1, 190]).iter().all(|a| *a > 0.99)); + // a square-pixel matte of the same size is pillarboxed, and Fit to Frame leaves it so + let square = matte(&mut s, (1, 1)); + let clip = overwrite_into_square_hd(&mut s, square); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + let c = coverage(&s, &[1, 20, 30, 96, 165, 175, 190]); + assert!(c[0] < 0.01 && c[1] < 0.01 && c[2] > 0.99 && c[3] > 0.99 && c[4] > 0.99 && c[5] < 0.01 && c[6] < 0.01, "{c:?}"); + s.execute("clip.fitToFrame", json!({"clips": [clip.0]})).unwrap(); + assert!((motion_scale(&s, clip) - 100.0).abs() < 1e-9); +} + +#[test] +fn default_media_scaling_uses_the_display_size() { + let mut s = Session::default(); + let item = matte(&mut s, (4, 3)); + // Set to Frame Size: 1920 x 1080 display in 1920 x 1080 needs no scaling + s.prefs.media.default_media_scaling = "setToFrameSize".into(); + let clip = overwrite_into_square_hd(&mut s, item); + assert_eq!(motion_scale(&s, clip), 100.0); + // in a 96 x 108 frame it is halved (display 192 wide), not scaled to 96 / 144 + s.execute("file.newSequence", json!({"name": "Narrow", "width": 96, "height": 108, "fps": 24})).unwrap(); + s.execute("source.open", json!({"item": item.0})).unwrap(); + let r = s.execute("source.overwrite", json!({})).unwrap(); + let clip = filmcraft_project::ClipId(r["clips"][0].as_u64().unwrap()); + assert_eq!(motion_scale(&s, clip), 50.0); + // Scale to Frame Size: the renderer fits the display size + s.prefs.media.default_media_scaling = "scaleToFrameSize".into(); + s.execute("file.newSequence", json!({"name": "Wide", "width": 384, "height": 108, "fps": 24})).unwrap(); + s.execute("source.open", json!({"item": item.0})).unwrap(); + s.execute("playhead.set", json!({"seconds": 0.0})).unwrap(); + s.execute("source.overwrite", json!({})).unwrap(); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + let c = coverage(&s, &[90, 100, 192, 283, 293]); + assert!(c[0] < 0.01 && c[1] > 0.99 && c[2] > 0.99 && c[3] > 0.99 && c[4] < 0.01, "192 wide, centred: {c:?}"); +} + +#[test] +fn interpret_footage_pixel_aspect_wins_over_the_file_and_undoes() { + let mut s = Session::default(); + let item = matte(&mut s, (4, 3)); + let r = s.execute("clip.interpretFootage", json!({"items": [item.0], "pixelAspect": [2, 1]})).unwrap(); + assert_eq!(r["pixelAspect"], json!([2, 1])); + let media = |s: &Session| s.project.item(item).unwrap().as_media().unwrap().clone(); + assert_eq!((media(&s).interpret.par, media(&s).pixel_aspect()), (Some((2, 1)), (2, 1))); + // the colour interpretation is left as it was + assert_eq!(media(&s).interpret.color_space, None); + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + assert_eq!(s.active_sequence().unwrap().settings.par, (2, 1)); + // a 2:1 picture 144 wide shows 288 wide: wider than the 192-wide square frame at 100 % + let clip = overwrite_into_square_hd(&mut s, item); + assert_eq!(motion_scale(&s, clip), 100.0); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + assert!(coverage(&s, &[1, 190]).iter().all(|a| *a > 0.99)); + s.execute("clip.fitToFrame", json!({"clips": [clip.0]})).unwrap(); + assert!((motion_scale(&s, clip) - 100.0 * 192.0 / 288.0).abs() < 1e-9); + // "file" goes back to the file's ratio; undo restores the override + s.execute("clip.interpretFootage", json!({"items": [item.0], "pixelAspect": "file"})).unwrap(); + assert_eq!((media(&s).interpret.par, media(&s).pixel_aspect()), (None, (4, 3))); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(media(&s).interpret.par, Some((2, 1))); + // hostile values are refused and change nothing + let rev = s.revision; + for bad in [ + json!([0, 1]), + json!([1, 0]), + json!([9, 1]), + json!([1, 9]), + json!([4]), + json!([4, 3, 1]), + json!([-4, 3]), + json!([4.5, 3]), + json!([u64::MAX, 1]), + json!("wide"), + json!({"n": 4}), + ] { + assert!(s.execute("clip.interpretFootage", json!({"items": [item.0], "pixelAspect": bad})).is_err(), "{bad}"); + } + assert!(s.execute("clip.interpretFootage", json!({"items": [item.0]})).is_err(), "nothing to change"); + assert_eq!((s.revision, media(&s).interpret.par), (rev, Some((2, 1)))); +} + +#[test] +fn hostile_file_pixel_aspects_make_square_sequences() { + for bad in [(0, 0), (0, 3), (4, 0), (u32::MAX, 1), (1, u32::MAX), (9, 1)] { + let mut s = Session::default(); + let item = matte(&mut s, bad); + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + let st = s.active_sequence().unwrap().settings.clone(); + assert_eq!(st.par, (1, 1), "{bad:?}"); + assert!(st.validate().is_ok()); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + assert!(coverage(&s, &[0, 143]).iter().all(|a| *a > 0.99), "{bad:?}"); + let clip = overwrite_into_square_hd(&mut s, item); + assert_eq!(motion_scale(&s, clip), 100.0); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + let c = coverage(&s, &[1, 96, 190]); + assert!(c[0] < 0.01 && c[1] > 0.99 && c[2] < 0.01, "{bad:?}: square, pillarboxed: {c:?}"); + } +} + +#[test] +fn generators_made_for_a_non_square_sequence_have_its_pixels() { + let mut s = Session::default(); + let item = matte(&mut s, (4, 3)); + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + // a Color Matte made now is 144 x 108 with the sequence's 4:3 pixels: it fills the frame + let r = s.execute("file.newColorMatte", json!({"color": "#00ff00"})).unwrap(); + let matte2 = ItemId(r["item"].as_u64().unwrap()); + let m = s.project.item(matte2).unwrap().as_media().unwrap(); + assert_eq!((m.info.video.as_ref().unwrap().width, m.pixel_aspect()), (144, (4, 3))); +} + +/// ffmpeg-made 144 x 108 H.264 with 4:3 samples (`setsar=4/3`): MP4 (`pasp`) and Matroska +/// (DisplayWidth / DisplayHeight). +fn anamorphic_fixtures() -> Option> { + let ffmpeg = filmcraft_testkit::oracle::ffmpeg_or_skip("par")?; + let dir = filmcraft_testkit::fixtures_dir("engine/par"); + let mut out = Vec::new(); + for ext in ["mp4", "mkv"] { + let path = dir.join(format!("anamorphic_144x108_sar43.{ext}")); + out.push(filmcraft_testkit::fixtures::generate(&path, |tmp| { + std::process::Command::new(&ffmpeg) + .args(["-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=144x108:rate=24:duration=1"]) + .args(["-vf", "setsar=4/3", "-c:v", "libx264", "-preset", "fast", "-pix_fmt", "yuv420p"]) + .arg(tmp) + .status() + .is_ok_and(|s| s.success()) + })?); + } + Some(out) +} + +#[test] +fn ffmpeg_anamorphic_files_fill_a_square_frame_and_make_anamorphic_sequences() { + let Some(files) = anamorphic_fixtures() else { return }; + for file in files { + let mut s = Session::default(); + let r = s.execute("file.import", json!({"paths": [file.to_string_lossy()]})).unwrap(); + let item = ItemId(r["items"][0].as_u64().unwrap()); + let v = s.project.item(item).unwrap().as_media().unwrap().info.video.clone().unwrap(); + assert_eq!((v.width, v.height, v.par), (144, 108, (4, 3)), "{}", file.display()); + s.execute("file.newSequenceFromClip", json!({"items": [item.0]})).unwrap(); + assert_eq!(s.active_sequence().unwrap().settings.par, (4, 3)); + overwrite_into_square_hd(&mut s, item); + s.execute("playhead.set", json!({"seconds": 0.5})).unwrap(); + // testsrc2 is opaque everywhere: covered edge to edge + let c = coverage(&s, &[1, 10, 181, 190]); + assert!(c.iter().all(|a| *a > 0.99), "{}: {c:?}", file.display()); + } +} diff --git a/crates/engine/src/proxies.rs b/crates/engine/src/proxies.rs index 4786f17b5..b4da00bee 100644 --- a/crates/engine/src/proxies.rs +++ b/crates/engine/src/proxies.rs @@ -103,6 +103,8 @@ pub fn transcode( if let Some(v) = &info.video { st.width = v.width; st.height = v.height; + // pixels in = pixels out: the sequence has the clip's pixel aspect, so nothing is stretched + st.par = clip.pixel_aspect(); } st.frame_rate = rate; st.sample_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate.max(8000)); diff --git a/crates/engine/src/settings.rs b/crates/engine/src/settings.rs index 1e0350358..598df6318 100644 --- a/crates/engine/src/settings.rs +++ b/crates/engine/src/settings.rs @@ -1211,17 +1211,18 @@ pub fn map_output(left: &[f32], right: &[f32], out: &mut [f32], ch: usize, map: } } -/// Settings ▸ Media ▸ Default Media Scaling for a clip of `src` size placed in a `frame`-sized -/// sequence: `scaleToFrameSize` turns on Scale to Frame Size (rasterised at frame size), -/// `setToFrameSize` sets Motion ▸ Scale so the picture fits the frame. -pub fn apply_media_scaling(ti: &mut filmcraft_project::TrackItem, scaling: &str, frame: (u32, u32), src: (u32, u32)) { - if src.0 == 0 || src.1 == 0 { +/// Settings ▸ Media ▸ Default Media Scaling for a clip placed in a `frame`-sized sequence. `src` +/// is the clip's size at its display aspect in sequence pixels +/// ([`filmcraft_project::conformed_size`]): `scaleToFrameSize` turns on Scale to Frame Size +/// (rasterised at frame size), `setToFrameSize` sets Motion ▸ Scale so the picture fits the frame. +pub fn apply_media_scaling(ti: &mut filmcraft_project::TrackItem, scaling: &str, frame: (u32, u32), src: (f64, f64)) { + if !(src.0 >= 1e-9 && src.1 >= 1e-9 && src.0.is_finite() && src.1.is_finite()) { return; } match scaling { "scaleToFrameSize" => ti.scale_to_frame = true, "setToFrameSize" => { - let fit = (frame.0 as f64 / src.0 as f64).min(frame.1 as f64 / src.1 as f64); + let fit = (frame.0 as f64 / src.0).min(frame.1 as f64 / src.1); if let Some(m) = ti.effects.iter_mut().find(|e| e.effect == "motion") { m.params.insert("scale".into(), filmcraft_project::Param::new(filmcraft_project::ParamValue::Float((fit * 1000.0).round() / 10.0))); } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index eec6c21f8..ddcafb700 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -247,6 +247,12 @@ impl MediaClip { pub fn frame_rate(&self) -> FrameRate { self.interpret.frame_rate.unwrap_or_else(|| self.info.frame_rate()) } + /// Pixel aspect ratio of the picture, in lowest terms: Modify ▸ Interpret Footage's override, + /// else the file's. A ratio that makes no sense ([`checked_par`]) is ignored: an override in + /// favour of the file's value, the file's in favour of square pixels. + pub fn pixel_aspect(&self) -> (u32, u32) { + self.interpret.par.and_then(checked_par).or_else(|| self.info.video.as_ref().and_then(|v| checked_par(v.par))).unwrap_or((1, 1)) + } pub fn duration(&self) -> Tick { self.info.duration } @@ -877,6 +883,51 @@ pub fn validate_frame_size(width: u32, height: u32) -> Result<(), String> { Ok(()) } +/// The most a pixel aspect ratio may stretch a picture either way (8:1 or 1:8). Real footage stays +/// well inside (2:1 for 2x anamorphic lenses, 4:3 for HDV, 0.9 for NTSC DV); a ratio beyond it is +/// garbage from a damaged file and is treated as square. +pub const MAX_PIXEL_ASPECT: f64 = 8.0; + +/// A pixel aspect ratio fit to compute with, in lowest terms, or `None` when it is unset (a zero +/// term, as decoders report "not signalled") or stretches more than [`MAX_PIXEL_ASPECT`] either +/// way. Ratios come from media, project and interchange files, so they are untrusted. +pub fn checked_par(par: (u32, u32)) -> Option<(u32, u32)> { + let (n, d) = par; + if n == 0 || d == 0 { + return None; + } + let r = f64::from(n) / f64::from(d); + if !(1.0 / MAX_PIXEL_ASPECT..=MAX_PIXEL_ASPECT).contains(&r) { + return None; + } + let (mut a, mut b) = (n, d); + while b != 0 { + (a, b) = (b, a % b); + } + let g = a.max(1); + Some((n / g, d / g)) +} + +/// [`checked_par`], with square pixels for a ratio that is unset or makes no sense. +pub fn sane_par(par: (u32, u32)) -> (u32, u32) { + checked_par(par).unwrap_or((1, 1)) +} + +/// The width of one pixel over its height ([`sane_par`] first): 1.333… for 4:3 pixels. +pub fn par_ratio(par: (u32, u32)) -> f64 { + let (n, d) = sane_par(par); + f64::from(n) / f64::from(d) +} + +/// The size of a `size` picture with `par` pixels in the pixels of a frame whose pixel aspect is +/// `frame_par`: the width scales by par / frame_par, the height stays. A 1440 x 1080 picture with +/// 4:3 pixels covers 1920 x 1080 square pixels. +pub fn conformed_size(size: (u32, u32), par: (u32, u32), frame_par: (u32, u32)) -> (f64, f64) { + let (par, frame_par) = (sane_par(par), sane_par(frame_par)); + let w = if par == frame_par { f64::from(size.0) } else { f64::from(size.0) * par_ratio(par) / par_ratio(frame_par) }; + (w, f64::from(size.1)) +} + impl SequenceSettings { pub fn validate(&self) -> Result<(), String> { validate_frame_size(self.width, self.height)?; @@ -1609,6 +1660,35 @@ impl Project { None } + /// Pixel aspect ratio of what an item shows, in lowest terms: a media clip's + /// ([`MediaClip::pixel_aspect`], Interpret Footage first), a nested sequence's, a subclip its + /// parent's. `None` for items drawn in the pixels of whatever sequence they are in (graphics, + /// adjustment layers) and for an item that doesn't exist. + pub fn source_par(&self, item: ItemId) -> Option<(u32, u32)> { + let mut id = item; + // subclips of subclips: bounded, as in `source_size` + for _ in 0..16 { + return match &self.item(id)?.kind { + ItemKind::Media(m) => Some(m.pixel_aspect()), + ItemKind::Sequence(q) => Some(sane_par(q.settings.par)), + ItemKind::AdjustmentLayer { .. } | ItemKind::Graphic { .. } => None, + ItemKind::Subclip { parent, .. } => { + id = *parent; + continue; + } + }; + } + None + } + + /// [`Project::source_size`] in the pixels of a frame whose pixel aspect is `frame_par` + /// ([`conformed_size`]): what Scale to Frame Size, Set to Frame Size and Fit / Fill fit into + /// the frame. Items without a pixel aspect of their own take the frame's. + pub fn conformed_source_size(&self, item: ItemId, frame_par: (u32, u32)) -> Option<(f64, f64)> { + let size = self.source_size(item)?; + Some(conformed_size(size, self.source_par(item).unwrap_or(frame_par), frame_par)) + } + /// Resolve the "auto" (NaN) points of the clips in the sequences `sequences` selects, for /// clips that did not go through a placing command (an interchange import builds them with /// the effect defaults). Points measured in the frame use the sequence's size. `anchor` is @@ -1950,4 +2030,61 @@ mod tests { nest.frame_hold = Some(f(100)); assert_eq!(p.nest_overhang(&nest), Some(nest.range())); } + + #[test] + fn pixel_aspect_ratios_are_reduced_and_hostile_ones_are_square() { + assert_eq!(checked_par((4, 3)), Some((4, 3))); + assert_eq!(checked_par((40, 30)), Some((4, 3))); + assert_eq!(checked_par((u32::MAX, u32::MAX)), Some((1, 1))); + assert_eq!(checked_par((u32::MAX, u32::MAX - 1)), Some((u32::MAX, u32::MAX - 1))); + assert_eq!(checked_par((8, 1)), Some((8, 1))); + assert_eq!(checked_par((1, 8)), Some((1, 8))); + for bad in [(0, 0), (0, 1), (1, 0), (9, 1), (1, 9), (u32::MAX, 1), (1, u32::MAX)] { + assert_eq!(checked_par(bad), None, "{bad:?}"); + assert_eq!(sane_par(bad), (1, 1), "{bad:?}"); + assert_eq!(par_ratio(bad), 1.0, "{bad:?}"); + } + assert!((par_ratio((4, 3)) - 4.0 / 3.0).abs() < 1e-12); + // 1440 x 1080 with 4:3 pixels is 1920 x 1080 square pixels, and itself in a 4:3 frame + assert_eq!(conformed_size((1440, 1080), (4, 3), (1, 1)), (1920.0, 1080.0)); + assert_eq!(conformed_size((1440, 1080), (4, 3), (8, 6)), (1440.0, 1080.0)); + assert_eq!(conformed_size((1920, 1080), (1, 1), (4, 3)), (1440.0, 1080.0)); + assert_eq!(conformed_size((u32::MAX, u32::MAX), (8, 1), (1, 8)), (u32::MAX as f64 * 64.0, u32::MAX as f64)); + assert_eq!(conformed_size((1440, 1080), (0, 0), (u32::MAX, 1)), (1440.0, 1080.0)); + } + + #[test] + fn source_par_follows_interpret_footage_subclips_and_sequences() { + let (mut p, clip, seq) = demo_project(); + assert_eq!(p.source_par(clip), Some((1, 1))); + let sub = p.add_item("Sub", Label::Iris, ItemKind::Subclip { parent: clip, range: TimeRange::new(Tick::ZERO, Tick(100)), restrict_trims: false }, None); + fn media(p: &mut Project, id: ItemId) -> &mut MediaClip { + match &mut p.item_mut(id).unwrap().kind { + ItemKind::Media(m) => m, + _ => unreachable!(), + } + } + // the file's ratio, reduced + media(&mut p, clip).info.video.as_mut().unwrap().par = (40, 30); + assert_eq!((p.source_par(clip), p.source_par(sub)), (Some((4, 3)), Some((4, 3)))); + assert_eq!(p.conformed_source_size(clip, (1, 1)), Some((1920.0 * 4.0 / 3.0, 1080.0))); + // Interpret Footage wins over the file + media(&mut p, clip).interpret.par = Some((2, 1)); + assert_eq!(p.source_par(sub), Some((2, 1))); + // a nonsensical override falls back to the file, a nonsensical file value to square + media(&mut p, clip).interpret.par = Some((0, 7)); + assert_eq!(p.source_par(clip), Some((4, 3))); + media(&mut p, clip).info.video.as_mut().unwrap().par = (u32::MAX, 1); + assert_eq!(p.source_par(clip), Some((1, 1))); + // a nested sequence has its own ratio; graphics and missing items have none + p.sequence_mut(seq).unwrap().settings.par = (0, 0); + assert_eq!(p.source_par(seq), Some((1, 1))); + p.sequence_mut(seq).unwrap().settings.par = (8, 6); + assert_eq!(p.source_par(seq), Some((4, 3))); + assert_eq!(p.conformed_source_size(seq, (1, 1)), Some((2560.0, 1080.0))); + let adj = p.add_item("Adj", Label::Iris, ItemKind::AdjustmentLayer { width: 1440, height: 1080, rate: FrameRate::FPS_24, duration: Tick(100) }, None); + assert_eq!(p.source_par(adj), None); + assert_eq!(p.conformed_source_size(adj, (4, 3)), Some((1440.0, 1080.0))); + assert_eq!(p.source_par(ItemId(987_654_321)), None); + } } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 780042189..1d6ff3c4e 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -282,7 +282,7 @@ pub(crate) fn render_seq_tracks(project: &Project, seq: &Sequence, t: Tick, opts /// it covers the whole output. fn adjustment_region(seq: &Sequence, item: &TrackItem, project: &Project, mt: Tick, scale: f32, w: usize, h: usize) -> Option { let size = source_size(project, item.item).unwrap_or((seq.settings.width, seq.settings.height)); - let motion = motion_matrix(seq, item, size, mt); + let motion = motion_matrix(seq, item, size, source_par(project, item.item), mt); let s = scale as f64; let (fw, fh) = (((size.0 as f64 * s).round() as usize).max(1), ((size.1 as f64 * s).round() as usize).max(1)); let m = Affine::scale(s, s).then_apply(&motion).then_apply(&Affine::scale(1.0 / s, 1.0 / s)); @@ -332,10 +332,26 @@ pub fn source_size(project: &Project, item: ItemId) -> Option<(u32, u32)> { project.source_size(item) } +/// Pixel aspect ratio of an item's source, for [`motion_matrix`]: Interpret Footage's override, +/// else the media's; `None` for items drawn in sequence pixels (graphics, adjustment layers). +pub fn source_par(project: &Project, item: ItemId) -> Option<(u32, u32)> { + project.source_par(item) +} + /// The Motion transform of an item at media time `mt`, mapping full-res source pixels to /// full-res sequence pixels. -pub fn motion_matrix(seq: &Sequence, item: &TrackItem, src: (u32, u32), mt: Tick) -> Affine { +/// +/// `src` is the source's size in its own (storage) pixels and `src_par` their pixel aspect ratio +/// (`None`: the sequence's, for graphics and adjustment layers). Motion works in square display +/// units, as Premiere's does: the Anchor Point is in source pixels and the Position in sequence +/// pixels, but Scale and Rotation see the picture at its display aspect, so a 1440 x 1080 clip with +/// 4:3 pixels covers a 1920 x 1080 square-pixel frame at 100 %, and a square-pixel clip in a 4:3 +/// sequence is narrowed to its display width. Scale to Frame Size fits the display size too. +pub fn motion_matrix(seq: &Sequence, item: &TrackItem, src: (u32, u32), src_par: Option<(u32, u32)>, mt: Tick) -> Affine { let (sw, sh) = (seq.settings.width as f64, seq.settings.height as f64); + let seq_par = filmcraft_project::sane_par(seq.settings.par); + let src_par = src_par.map_or(seq_par, filmcraft_project::sane_par); + let (p, q) = (filmcraft_project::par_ratio(src_par), filmcraft_project::par_ratio(seq_par)); let mut pos = Vec2::new(sw / 2.0, sh / 2.0); let mut anchor = Vec2::new(src.0 as f64 / 2.0, src.1 as f64 / 2.0); let mut scale = Vec2::new(1.0, 1.0); @@ -356,10 +372,22 @@ pub fn motion_matrix(seq: &Sequence, item: &TrackItem, src: (u32, u32), mt: Tick rot = m.f64_at("rotation", mt); } if item.scale_to_frame { - let fit = (sw / src.0 as f64).min(sh / src.1 as f64); + // the display size fitted in the frame's display size + let fit = (sw * q / (src.0.max(1) as f64 * p)).min(sh / src.1.max(1) as f64); scale = scale * fit; } - Affine::motion(pos, scale, rot, anchor) + // source px → (× p) display units → Scale, Rotation → (÷ q) sequence px. Without rotation the + // two stretches merge into one horizontal factor, exactly 1 when the ratios are the same. + if rot == 0.0 { + let r = if src_par == seq_par { 1.0 } else { p / q }; + return Affine::motion(pos, Vec2::new(scale.x * r, scale.y), rot, anchor); + } + Affine::translate(pos.x, pos.y).then_apply(&Affine::scale(1.0 / q, 1.0)).then_apply(&Affine::motion( + Vec2::ZERO, + Vec2::new(scale.x * p, scale.y), + rot, + anchor, + )) } /// Render one track item's layer at timeline time `t` into a canvas-sized image. @@ -403,7 +431,7 @@ pub(crate) fn item_layer_ex( // time inside a frame hold without Hold Filters). let mt = item.effect_time_at(t); let src_size = source_size(project, item.item)?; - let motion = motion_matrix(seq, item, src_size, mt); + let motion = motion_matrix(seq, item, src_size, source_par(project, item.item), mt); // How many output pixels one source pixel covers → request a reduced frame when possible. let lin = ((motion.a * motion.a + motion.b * motion.b).sqrt()).max((motion.c * motion.c + motion.d * motion.d).sqrt()); let want = (lin * opts.scale as f64).clamp(1.0 / 64.0, 1.0) as f32; @@ -678,3 +706,7 @@ mod preview_tests; #[cfg(test)] #[path = "region_tests.rs"] mod region_tests; + +#[cfg(test)] +#[path = "par_tests.rs"] +mod par_tests; diff --git a/crates/render/src/par_tests.rs b/crates/render/src/par_tests.rs new file mode 100644 index 000000000..bca5931c2 --- /dev/null +++ b/crates/render/src/par_tests.rs @@ -0,0 +1,270 @@ +//! Pixel aspect ratio: a clip with non-square pixels is placed at its display aspect (a 1440 x 1080 +//! picture with 4:3 pixels covers a 1920 x 1080 square-pixel frame), a sequence with non-square +//! pixels holds clips at its own aspect, Interpret Footage overrides the file's ratio, and hostile +//! ratios are square. Small in-memory frames stand in for 1440 x 1080 / 1920 x 1080 at a tenth of +//! the size: 144 x 108 with 4:3 pixels shows 192 x 108. + +use super::*; +use filmcraft_frame::AudioBuffer; +use filmcraft_media::{MediaInfo, MediaKind, MediaSource, VideoStreamInfo}; +use filmcraft_project::{Label, MediaClip, MediaRef, SequenceSettings, TrackKind, resolve_auto_points}; +use filmcraft_time::{FrameRate, TICKS_PER_SECOND, TimeRange}; + +const FPS: FrameRate = FrameRate::FPS_24; + +/// One still picture: the left half red, the right half blue (opaque). +struct Halves { + info: MediaInfo, + frame: Arc, +} + +impl MediaSource for Halves { + fn info(&self) -> &MediaInfo { + &self.info + } + fn video_frame(&self, _req: FrameRequest) -> filmcraft_media::Result> { + Ok(self.frame.clone()) + } + fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { + Ok(AudioBuffer::silence(sample_rate, 2, frames)) + } +} + +fn halves(w: u32, h: u32, par: (u32, u32)) -> Halves { + let px: Vec = (0..w * h).flat_map(|i| if i % w < w / 2 { [255, 0, 0, 255] } else { [0, 0, 255, 255] }).collect(); + let frame = Arc::new(VideoFrame { par, ..VideoFrame::rgba8(w, h, px) }); + let info = MediaInfo { + name: "halves".into(), + kind: MediaKind::Movie, + duration: Tick(10 * TICKS_PER_SECOND), + video: Some(VideoStreamInfo { + width: w, + height: h, + frame_rate: FPS, + par, + codec: "test".into(), + pixel_format: "RGBA 8-bit".into(), + color: filmcraft_color::ColorInfo::SRGB_FULL, + has_alpha: false, + bitrate: None, + hdr: None, + }), + audio: None, + container: "test".into(), + start_timecode: None, + file_size: None, + }; + Halves { info, frame } +} + +/// A project with a `src`-sized clip of `src_par` pixels on V1 of a `seq`-sized sequence with +/// `seq_par` pixels, placed as the timeline places media (Motion's points resolved, 100 %). +fn project(src: (u32, u32), src_par: (u32, u32), seq: (u32, u32), seq_par: (u32, u32), scale_to_frame: bool) -> (Project, ItemId, ItemId, SourceMap) { + let mut p = Project::new("par"); + let source = halves(src.0, src.1, src_par); + let clip = MediaClip { + media: MediaRef::File { path: "halves.mov".into() }, + info: source.info.clone(), + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }; + let item = p.add_item("halves", Label::Iris, ItemKind::Media(clip), None); + let settings = SequenceSettings { width: seq.0, height: seq.1, par: seq_par, frame_rate: FPS, ..Default::default() }; + let sid = p.new_sequence("s", settings, 1, 0, None); + let mut ti = p.make_track_item(item, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, FPS.tick_of(24)), FPS).unwrap(); + for e in &mut ti.effects { + resolve_auto_points(e, seq, src); + } + ti.scale_to_frame = scale_to_frame; + p.sequence_mut(sid).unwrap().video_tracks[0].items.push(ti); + let mut map = SourceMap::default(); + map.0.insert(item, Arc::new(source) as SharedSource); + (p, sid, item, map) +} + +fn render(p: &Project, seq: ItemId, map: &SourceMap) -> Image { + render_sequence(p, seq, Tick::ZERO, RenderOptions::default(), map) +} + +/// Middle-row colour of column `x`: 'r', 'b' or '.' (transparent). +fn col(img: &Image, x: usize) -> char { + let c = img.get(x, img.h / 2); + match c { + _ if c[3] < 0.01 => '.', + _ if c[3] > 0.99 && c[0] > 0.99 && c[2] < 0.01 => 'r', + _ if c[3] > 0.99 && c[2] > 0.99 && c[0] < 0.01 => 'b', + _ => '?', + } +} + +/// The middle row as a string of `col`s, one per column. +fn row(img: &Image) -> String { + (0..img.w).map(|x| col(img, x)).collect() +} + +/// `got` is `want` except where bilinear filtering mixes neighbours: next to a change in `want` +/// and at the frame's edges. +#[track_caller] +fn assert_row(got: &str, want: &str) { + let (g, w): (Vec, Vec) = (got.chars().collect(), want.chars().collect()); + assert_eq!(g.len(), w.len(), "\n got {got}\nwant {want}"); + for i in 1..w.len().saturating_sub(1) { + if w[i - 1] == w[i] && w[i + 1] == w[i] { + assert_eq!(g[i], w[i], "column {i}\n got {got}\nwant {want}"); + } + } +} + +#[test] +fn an_anamorphic_clip_fills_a_square_pixel_frame_at_100_percent() { + // 1440 x 1080 with 4:3 pixels in 1920 x 1080: the whole frame, split in the middle + let (p, seq, _, map) = project((144, 108), (4, 3), (192, 108), (1, 1), false); + let img = render(&p, seq, &map); + assert_row(&row(&img), &format!("{}{}", "r".repeat(96), "b".repeat(96))); + // every row is covered, top to bottom + for y in [1, 53, 106] { + assert!(img.get(1, y)[3] > 0.99 && img.get(190, y)[3] > 0.99, "row {y}"); + } + // the same picture with square pixels is pillarboxed at 1440 (144) wide: what the bug showed + let (p, seq, _, map) = project((144, 108), (1, 1), (192, 108), (1, 1), false); + assert_row(&row(&render(&p, seq, &map)), &format!("{}{}{}{}", ".".repeat(24), "r".repeat(72), "b".repeat(72), ".".repeat(24))); +} + +#[test] +fn the_gpu_plan_places_an_anamorphic_clip_like_the_cpu() { + let (p, seq, _, map) = project((144, 108), (4, 3), (192, 108), (1, 1), false); + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + let want = render_sequence(&p, seq, Tick::ZERO, opts, &map); + let got = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, opts, &map)); + assert_eq!((got.w, got.h), (want.w, want.h)); + assert_row(&row(&got), &row(&want)); + } +} + +#[test] +fn scale_to_frame_size_fits_the_display_size() { + // 1920-wide display picture in a 3840 x 1080 frame: fitted by height at 1:1, 1920 wide + let (p, seq, _, map) = project((144, 108), (4, 3), (384, 108), (1, 1), true); + assert_row(&row(&render(&p, seq, &map)), &format!("{}{}{}{}", ".".repeat(96), "r".repeat(96), "b".repeat(96), ".".repeat(96))); + // fitted by width: 1920 x 1080 display in 960 x 1080 → 960 x 540, letterboxed + let (p, seq, _, map) = project((144, 108), (4, 3), (96, 108), (1, 1), true); + let img = render(&p, seq, &map); + assert_row(&row(&img), &format!("{}{}", "r".repeat(48), "b".repeat(48))); + assert!(img.get(48, 10)[3] < 0.01 && img.get(48, 54)[3] > 0.99, "letterboxed to 54 rows"); +} + +#[test] +fn a_sequence_with_the_clips_pixel_aspect_shows_it_pixel_for_pixel() { + // New Sequence From Clip: 1440 x 1080, 4:3 pixels; the clip maps 1:1 onto the frame + let (p, seq, _, map) = project((144, 108), (4, 3), (144, 108), (4, 3), false); + let m = motion_matrix(p.sequence(seq).unwrap(), &p.sequence(seq).unwrap().video_tracks[0].items[0], (144, 108), Some((4, 3)), Tick::ZERO); + assert!(identity_placement(&m), "{m:?}"); + let img = render(&p, seq, &map); + assert_eq!(row(&img), format!("{}{}", "r".repeat(72), "b".repeat(72))); + // the same ratio written differently is the same ratio + let (p, seq, _, map) = project((144, 108), (8, 6), (144, 108), (4, 3), false); + assert_eq!(render(&p, seq, &map).px, img.px); +} + +#[test] +fn square_pixel_clips_are_narrowed_in_a_non_square_sequence() { + // 1920 x 1080 square in 1440 x 1080 with 4:3 pixels: exactly the frame + let (p, seq, _, map) = project((192, 108), (1, 1), (144, 108), (4, 3), false); + assert_row(&row(&render(&p, seq, &map)), &format!("{}{}", "r".repeat(72), "b".repeat(72))); + // 1440 x 1080 square (4:3 picture) in it: 1080 sequence pixels wide, pillarboxed + let (p, seq, _, map) = project((144, 108), (1, 1), (144, 108), (4, 3), false); + assert_row(&row(&render(&p, seq, &map)), &format!("{}{}{}{}", ".".repeat(18), "r".repeat(54), "b".repeat(54), ".".repeat(18))); +} + +#[test] +fn rotation_happens_at_the_display_aspect() { + // a 100 x 100 square-pixel source in a 200 x 100 sequence with 2:1 pixels shows 50 x 100 + // sequence pixels (a square on screen); turned 90° it is still that square + let settings = SequenceSettings { width: 200, height: 100, par: (2, 1), ..Default::default() }; + let mut p = Project::new("rot"); + let sid = p.new_sequence("s", settings, 1, 0, None); + let adj = p.add_item("a", Label::Iris, ItemKind::AdjustmentLayer { width: 100, height: 100, rate: FPS, duration: Tick(TICKS_PER_SECOND) }, None); + let mut ti = p.make_track_item(adj, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, FPS.tick_of(1)), FPS).unwrap(); + for e in &mut ti.effects { + resolve_auto_points(e, (200, 100), (100, 100)); + } + let seq = p.sequence(sid).unwrap(); + let r = filmcraft_geom::Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 }; + for rot in [0.0, 90.0, -270.0] { + ti.effect_mut("motion").unwrap().params.get_mut("rotation").unwrap().value = ParamValue::Float(rot); + let b = motion_matrix(seq, &ti, (100, 100), Some((1, 1)), Tick::ZERO).bounds(&r); + assert!((b.w - 50.0).abs() < 1e-9 && (b.h - 100.0).abs() < 1e-9 && (b.x - 75.0).abs() < 1e-9, "{rot}°: {b:?}"); + } + // a 45° turn: the square's diagonal, √2 · 100 display units, both ways + ti.effect_mut("motion").unwrap().params.get_mut("rotation").unwrap().value = ParamValue::Float(45.0); + let b = motion_matrix(seq, &ti, (100, 100), Some((1, 1)), Tick::ZERO).bounds(&r); + let d = 100.0 * std::f64::consts::SQRT_2; + assert!((b.w - d / 2.0).abs() < 1e-9 && (b.h - d).abs() < 1e-9, "{b:?}"); + // graphics and adjustment layers (no ratio of their own) are in sequence pixels + ti.effect_mut("motion").unwrap().params.get_mut("rotation").unwrap().value = ParamValue::Float(0.0); + let b = motion_matrix(seq, &ti, (100, 100), None, Tick::ZERO).bounds(&r); + assert!((b.w - 100.0).abs() < 1e-9, "{b:?}"); +} + +#[test] +fn interpret_footage_overrides_the_files_pixel_aspect() { + // the file says square; Interpret Footage says 4:3 + let (mut p, seq, item, map) = project((144, 108), (1, 1), (192, 108), (1, 1), false); + match &mut p.item_mut(item).unwrap().kind { + ItemKind::Media(m) => m.interpret.par = Some((4, 3)), + _ => unreachable!(), + } + assert_row(&row(&render(&p, seq, &map)), &format!("{}{}", "r".repeat(96), "b".repeat(96))); + // and the other way round: a 4:3 file interpreted as square is pillarboxed + let (mut p, seq, item, map) = project((144, 108), (4, 3), (192, 108), (1, 1), false); + match &mut p.item_mut(item).unwrap().kind { + ItemKind::Media(m) => m.interpret.par = Some((1, 1)), + _ => unreachable!(), + } + assert!(row(&render(&p, seq, &map)).starts_with(&".".repeat(24))); +} + +#[test] +fn hostile_pixel_aspect_ratios_render_square_without_panicking() { + let pillarboxed = format!("{}{}{}{}", ".".repeat(24), "r".repeat(72), "b".repeat(72), ".".repeat(24)); + let filled = format!("{}{}", "r".repeat(96), "b".repeat(96)); + let hostile = [(0, 0), (0, 3), (4, 0), (u32::MAX, 1), (1, u32::MAX), (9, 1), (1, 9)]; + for bad in hostile { + // a clip's ratio: square + let r = std::panic::catch_unwind(|| { + let (p, seq, _, map) = project((144, 108), bad, (192, 108), (1, 1), false); + row(&render(&p, seq, &map)) + }); + assert_row(&r.unwrap_or_default(), &pillarboxed); + // a sequence's ratio (a damaged project): square, so a 4:3-pixel clip fills it + let r = std::panic::catch_unwind(|| { + let (p, seq, _, map) = project((144, 108), (4, 3), (192, 108), bad, true); + let plan = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, RenderOptions::default(), &map)); + (row(&render(&p, seq, &map)), row(&plan)) + }); + let (cpu, gpu) = r.unwrap_or_default(); + assert_row(&cpu, &filled); + assert_row(&gpu, &filled); + // an Interpret Footage override that makes no sense: the file's ratio + let r = std::panic::catch_unwind(|| { + let (mut p, seq, item, map) = project((144, 108), (4, 3), (192, 108), (1, 1), false); + if let ItemKind::Media(m) = &mut p.item_mut(item).unwrap().kind { + m.interpret.par = Some(bad); + } + row(&render(&p, seq, &map)) + }); + assert_row(&r.unwrap_or_default(), &filled); + } + // extreme but valid ratios on degenerate sizes stay finite + for (src, par) in [((1, 1), (8, 1)), ((1, 32_768), (1, 8)), ((32_768, 1), (8, 1))] { + let (p, seq, _, map) = project(src, par, (16, 16), (1, 8), true); + let img = render(&p, seq, &map); + assert!(img.px.iter().all(|v| v.is_finite()), "{src:?} {par:?}"); + } +} diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs index 628e7e102..f986711c6 100644 --- a/crates/render/src/plan.rs +++ b/crates/render/src/plan.rs @@ -340,7 +340,7 @@ fn push_item( && let Some(angle) = item.multicam_angle(nested) && !(opts.effects && item.has_standard_effects()) && (nested.settings.width, nested.settings.height) == (seq.settings.width, seq.settings.height) - && near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), mt)) + && near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), Some(nested.settings.par), mt)) && let Some(tr) = nested.angle_video_track_index(angle).and_then(|i| nested.video_tracks.get(i)) && !tr.transitions.iter().any(|x| x.range().contains(ft)) && tr.item_at(ft).is_none_or(|i| crate::opacity_blend(i, i.effect_time_at(ft)).1 != Blend::Dissolve) @@ -362,7 +362,7 @@ fn push_item( && !item.has_opacity_masks() && bl == Blend::Normal && extra_opacity * op >= 1.0 - 1e-6 - && near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), mt)) + && near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), Some(nested.settings.par), mt)) && nest_is_plain(project, seq, nested, ft) { push_tracks(project, nested, ft, opts, sources, nest + 1, out); @@ -384,7 +384,7 @@ fn push_item( { let Some(size) = crate::source_size(project, item.item) else { return }; let (w, h) = output_size(seq, opts.scale); - let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion_matrix(seq, item, size, mt)); + let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion_matrix(seq, item, size, None, mt)); if let Some((img, x, y)) = crate::graphic_clip::render_graphic_tight(item, mt, size, &m, w, h) { out.push(PlanLayer::new(cpu_frame(img), Affine::translate(x as f64, y as f64), op * extra_opacity, bl)); } @@ -393,7 +393,7 @@ fn push_item( if let Some(chain) = gpu_chain(project, item, opts) { let Some(src) = sources.source(item.item) else { return }; let Some(size) = crate::source_size(project, item.item) else { return }; - let motion = motion_matrix(seq, item, size, mt); + let motion = motion_matrix(seq, item, size, crate::source_par(project, item.item), mt); let lin = ((motion.a * motion.a + motion.b * motion.b).sqrt()).max((motion.c * motion.c + motion.d * motion.d).sqrt()); let want = (lin * opts.scale as f64).clamp(1.0 / 64.0, 1.0) as f32; let time = crate::video_source_time(item, t, src.info().frame_rate()); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index d60469357..bf18d0069 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -411,7 +411,7 @@ pub fn invoke(app: &mut FilmcraftApp, ctx: &egui::Context, id: &str, mut params: return Ok(json!({"dialog": "audioGain"})); } // Colour dialogs from the menus; with params the engine command applies directly. - "clip.interpretFootage" if params.get("colorSpace").is_none() => { + "clip.interpretFootage" if params.get("colorSpace").is_none() && params.get("pixelAspect").is_none() => { filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; crate::panels::color_dialogs::open_interpret(app, ¶ms); return Ok(json!({"dialog": "interpretFootage"})); diff --git a/crates/ui-egui/src/panels/graphics.rs b/crates/ui-egui/src/panels/graphics.rs index 46feeb4ce..2b2065e0e 100644 --- a/crates/ui-egui/src/panels/graphics.rs +++ b/crates/ui-egui/src/panels/graphics.rs @@ -87,11 +87,17 @@ pub fn selected_graphic(app: &FilmcraftApp) -> Option<(ClipId, TrackItem)> { /// Layers of the graphic clips visible at the playhead, front-most first. A layer whose /// visibility is off is not in the picture, so it has no box and cannot be clicked either. +/// Sequence pixels → screen points on the monitor picture `pic`. The picture shows the frame at its +/// display aspect, so with non-square sequence pixels the two axes scale differently. +fn frame_to_screen(pic: Rect, frame: (u32, u32)) -> Affine { + let (kx, ky) = (pic.width() as f64 / frame.0.max(1) as f64, pic.height() as f64 / frame.1.max(1) as f64); + Affine::translate(pic.min.x as f64, pic.min.y as f64).then_apply(&Affine::scale(kx, ky)) +} + pub fn visible_layers(app: &FilmcraftApp, pic: Rect, frame: (u32, u32)) -> Vec { let Some(seq) = app.session.active_sequence() else { return Vec::new() }; let t = app.session.playhead(); - let k = pic.width() as f64 / frame.0.max(1) as f64; - let screen = Affine::translate(pic.min.x as f64, pic.min.y as f64).then_apply(&Affine::scale(k, k)); + let screen = frame_to_screen(pic, frame); let mut out = Vec::new(); for tr in seq.video_tracks.iter().rev() { if !tr.enabled { @@ -104,7 +110,7 @@ pub fn visible_layers(app: &FilmcraftApp, pic: Rect, frame: (u32, u32)) -> Vec Vec (Option, Affine) { - let k = pic.width() as f64 / frame.0.max(1) as f64; - let screen = Affine::translate(pic.min.x as f64, pic.min.y as f64).then_apply(&Affine::scale(k, k)); + let screen = frame_to_screen(pic, frame); let t = app.session.playhead(); if let Some((c, it)) = selected_graphic(app) && it.range().contains(t) && let Some(seq) = app.session.active_sequence() && let Some(ItemKind::Graphic { width, height, .. }) = app.session.project.item(it.item).map(|p| &p.kind) { - let m = screen.then_apply(&filmcraft_render::motion_matrix(seq, &it, (*width, *height), it.source_time_at(t))); + let m = screen.then_apply(&filmcraft_render::motion_matrix(seq, &it, (*width, *height), None, it.source_time_at(t))); return (Some(c), m); } (None, screen) diff --git a/crates/ui-egui/src/panels/masks.rs b/crates/ui-egui/src/panels/masks.rs index 1d50f9ab0..4ffba906a 100644 --- a/crates/ui-egui/src/panels/masks.rs +++ b/crates/ui-egui/src/panels/masks.rs @@ -163,7 +163,7 @@ pub fn effect_rows( fn clip_to_screen(app: &FilmcraftApp, it: &TrackItem, mt: Tick, pic: Rect, frame: (u32, u32)) -> Option { let seq = app.session.active_sequence()?; let size = filmcraft_render::source_size(&app.session.project, it.item).unwrap_or(frame); - let motion = filmcraft_render::motion_matrix(seq, it, size, mt); + let motion = filmcraft_render::motion_matrix(seq, it, size, filmcraft_render::source_par(&app.session.project, it.item), mt); let view = Affine::translate(pic.min.x as f64, pic.min.y as f64) .then_apply(&Affine::scale(pic.width() as f64 / frame.0 as f64, pic.height() as f64 / frame.1 as f64)); Some(view.then_apply(&motion)) diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index 39034bfaa..c87a90c6a 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -157,12 +157,13 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) } else { (None, video_area) }; - // ---- picture + // ---- picture, at its display aspect (non-square pixels are stretched on screen) let ppp = ctx.pixels_per_point(); + let display_w = (frame_size.0 as f64 * monitor_view::pixel_aspect(app, which)) as f32; let pic = if compare { - fit(video_area, frame_size.0 as f32, frame_size.1 as f32) + fit(video_area, display_w, frame_size.1 as f32) } else if show_picture { - monitor_view::picture_rect(video_area, frame_size.0 as f32, frame_size.1 as f32, &mv, ppp) + monitor_view::picture_rect(video_area, display_w, frame_size.1 as f32, &mv, ppp) } else { video_area }; @@ -172,7 +173,8 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) ui.painter().rect_filled(pic, 0.0, t.monitor_bg); let playing = if which == Which::Program { app.playback.playing } else { app.source_playback.clock.playing }; let res = mv.effective_res(playing); - let screen_scale = (pic.width() * ppp / frame_size.0 as f32).min(1.0); + // frame pixels per screen pixel along the axis that needs the most of them + let screen_scale = (pic.width() * ppp / frame_size.0.max(1) as f32).max(pic.height() * ppp / frame_size.1.max(1) as f32).min(1.0); let scale = quantize_scale(res.scale().min(screen_scale.max(1.0 / 32.0))); let frame = rate.frame_at(time); let rev = match target { @@ -240,7 +242,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) let rf = rate.frame_at(Tick(mv.compare_ref.unwrap_or(time.0))); let rkey = FrameKey { target: cpu_target, frame: rf, size: size_key, revision: rev, draft: false }; app.frames.request(rkey, rate.tick_of(rf), scale, &project, 1); - let rpic = fit(ra, frame_size.0 as f32, frame_size.1 as f32); + let rpic = fit(ra, display_w, frame_size.1 as f32); ui.painter().rect_filled(rpic, 0.0, t.monitor_bg); if let (Some(tex), _) = cpu_texture(app, &ctx, &format!("monitor-{prefix}-ref"), rkey, &[], DisplayMode::Composite) { ui.painter().image(tex, rpic, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); diff --git a/crates/ui-egui/src/panels/monitor_view.rs b/crates/ui-egui/src/panels/monitor_view.rs index 0f866a1ec..dd3db365a 100644 --- a/crates/ui-egui/src/panels/monitor_view.rs +++ b/crates/ui-egui/src/panels/monitor_view.rs @@ -97,6 +97,17 @@ fn frame_size(app: &FilmcraftApp, w: Which) -> Option<(u32, u32)> { } } +/// Width over height of one pixel of a monitor's picture: the active sequence's pixel aspect +/// ratio (Program) or the Source item's (Interpret Footage first). Monitors show the picture at +/// its display aspect, so a 1440 x 1080 frame with 4:3 pixels is drawn 16:9. +pub fn pixel_aspect(app: &FilmcraftApp, w: Which) -> f64 { + let par = match w { + Which::Program => app.session.active_sequence().map(|q| q.settings.par), + Which::Source => app.session.state.source_item.and_then(|i| app.session.project.source_par(i)), + }; + par.map_or(1.0, filmcraft_project::par_ratio) +} + /// Handle a `view.*` command (None for other ids). pub fn route(app: &mut FilmcraftApp, id: &str, params: &Value) -> Option> { let rest = id.strip_prefix("view.")?; diff --git a/crates/ui-egui/src/panels/sequence_settings.rs b/crates/ui-egui/src/panels/sequence_settings.rs index 7c3e77a31..d7fe5d025 100644 --- a/crates/ui-egui/src/panels/sequence_settings.rs +++ b/crates/ui-egui/src/panels/sequence_settings.rs @@ -12,7 +12,7 @@ //! //! Premiere settings FilmCraft has no equivalent for yet are shown greyed out with their current //! value and a "not supported yet" hint, rather than editable and silently ignored: Editing Mode, -//! Pixel Aspect Ratio (stored, not rendered), Fields, the Frames and Feet + Frames display +//! Pixel Aspect Ratio (rendered, and set by New Sequence From Clip, but not editable here yet), Fields, the Frames and Feet + Frames display //! formats, Number of Channels, audio Display Format, the Video Previews format, codec and size, //! Maximum Bit Depth, and the VR Properties tab. Composite in Linear Color is shown on and greyed: //! FilmCraft always composites in linear light. @@ -174,6 +174,12 @@ fn aspect(w: u32, h: u32) -> String { } } +/// The width a `w` x `h` frame with `par` pixels shows at, in square pixels (rounded). +fn display_width(w: u32, h: u32, par: (u32, u32)) -> u32 { + let dw = filmcraft_project::conformed_size((w, h), par, (1, 1)).0.round(); + if dw >= f64::from(u32::MAX) { u32::MAX } else { dw.max(0.0) as u32 } +} + fn gcd(mut a: u32, mut b: u32) -> u32 { while b != 0 { (a, b) = (b, a % b); @@ -331,7 +337,8 @@ fn general(ui: &mut egui::Ui, d: &mut SequenceSettingsDraft, cur: &SequenceSetti let h = ui.add(egui::DragValue::new(&mut d.height).range(1..=filmcraft_project::MAX_FRAME_SIDE).speed(1.0)); elems.push(("sequenceSettings.height".into(), h.rect, d.height.to_string())); ui.label(tl!("vertical")); - let a = aspect(d.width, d.height); + // the display aspect: 1440 x 1080 with 4:3 pixels is 16:9 + let a = aspect(display_width(d.width, d.height, cur.par), d.height); let ar = ui.label(&a); elems.push(("sequenceSettings.aspect".into(), ar.rect, a)); }); @@ -562,6 +569,11 @@ mod tests { assert_eq!(aspect(1080, 1920), "9:16"); assert_eq!(aspect(1440, 1080), "4:3"); assert_eq!(aspect(4096, 2160), "1.90:1"); + // non-square pixels: the display aspect + assert_eq!(aspect(display_width(1440, 1080, (4, 3)), 1080), "16:9"); + assert_eq!(aspect(display_width(1280, 1080, (3, 2)), 1080), "16:9"); + assert_eq!(display_width(u32::MAX, 1, (8, 1)), u32::MAX); + assert_eq!(display_width(1440, 1080, (0, 0)), 1440); } #[test] diff --git a/crates/ui-egui/tests/par_ui.rs b/crates/ui-egui/tests/par_ui.rs new file mode 100644 index 000000000..c656b2456 --- /dev/null +++ b/crates/ui-egui/tests/par_ui.rs @@ -0,0 +1,111 @@ +//! Monitors show pictures at their display aspect: a 1440 x 1080 clip with 4:3 pixels (here a +//! 144 x 108 Color Matte given 4:3 pixels) is drawn 16:9 in the Source monitor, and so is the +//! sequence New Sequence From Clip makes of it in the Program monitor. Square pixels stay 4:3. +//! Driven headless through the control channel. +//! +//! With `FILMCRAFT_UI_SHOTS=` the test also renders the UI with wgpu and writes `par-*.png`. + +use std::sync::Arc; +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_engine::project::{ItemId, ItemKind}; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, + shots: Option, +} + +impl Driver { + fn new(session: Session) -> Self { + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let shots = std::env::var_os("FILMCRAFT_UI_SHOTS").map(std::path::PathBuf::from); + let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000); + if shots.is_some() { + b = b.wgpu().with_pixels_per_point(1.0); + } + let harness = b.build_eframe(move |_cc| app); + let mut d = Driver { harness, tx, shots }; + d.frames(4); + d + } + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + fn ok(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(r) = reply.try_recv() { + assert_eq!(r["ok"], true, "{method} {params}: {r}"); + return r["result"].clone(); + } + } + panic!("no reply to {method} {params}"); + } + fn exec(&mut self, command: &str, params: Value) -> Value { + let r = self.ok("engine.execute", json!({"command": command, "params": params})); + self.frames(3); + r + } + fn shot(&mut self, name: &str) { + let Some(dir) = self.shots.clone() else { return }; + // give the frame workers time to deliver the pictures + for _ in 0..8 { + self.frames(3); + std::thread::sleep(std::time::Duration::from_millis(150)); + } + let img = self.harness.render().expect("wgpu render"); + std::fs::create_dir_all(&dir).unwrap(); + img.save(dir.join(format!("par-{name}.png"))).unwrap(); + } + /// Width over height of an element's rect. + fn aspect(&mut self, id: &str) -> f64 { + let v = self.ok("ui.elements", json!({"prefix": id})); + let e = v.as_array().unwrap().iter().find(|e| e["id"] == id).unwrap_or_else(|| panic!("no element {id}")).clone(); + let r: Vec = e["rect"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap()).collect(); + r[2] / r[3] + } +} + +/// A session with a 144 x 108 Color Matte whose pixels are `par`. +fn session_with_matte(par: (u32, u32)) -> (Session, ItemId) { + let mut s = Session::default(); + let r = s.execute("file.newColorMatte", json!({"color": "#3366ff", "width": 144, "height": 108, "seconds": 2.0})).unwrap(); + let id = ItemId(r["item"].as_u64().unwrap()); + if let Some(ItemKind::Media(m)) = Arc::make_mut(&mut s.project).item_mut(id).map(|i| &mut i.kind) { + m.info.video.as_mut().unwrap().par = par; + } + (s, id) +} + +#[test] +fn monitors_show_non_square_pixels_at_the_display_aspect() { + for (par, want) in [((4, 3), 16.0 / 9.0), ((1, 1), 4.0 / 3.0), ((0, 0), 4.0 / 3.0), ((u32::MAX, 1), 4.0 / 3.0)] { + let (s, item) = session_with_matte(par); + let mut d = Driver::new(s); + d.exec("file.newSequenceFromClip", json!({"items": [item.0]})); + d.exec("source.open", json!({"item": item.0})); + let (src, prg) = (d.aspect("source.picture"), d.aspect("program.picture")); + assert!((src - want).abs() < 0.01, "{par:?}: Source picture {src}"); + assert!((prg - want).abs() < 0.01, "{par:?}: Program picture {prg}"); + d.shot(&format!("monitors-{}x{}", par.0, par.1)); + // Interpret Footage changes the Source monitor's picture at once + d.exec("clip.interpretFootage", json!({"items": [item.0], "pixelAspect": [2, 1]})); + let src = d.aspect("source.picture"); + assert!((src - 8.0 / 3.0).abs() < 0.01, "{par:?}: interpreted 2:1, Source picture {src}"); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 055033cb0..967fb160b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -220,6 +220,16 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see `audio(start, frames, rate)` in media time. Sources are `Send + Sync` and shared by monitors, thumbnails, playback and export. The engine's `MediaPool` creates one per project item, lazily, through registered openers (`codecs::openers()`: MP4/MOV, MKV/WebM, audio files). +- **Pixel aspect ratio.** Demuxers report it in `VideoStreamInfo::par` (MP4/MOV `pasp`, Matroska + display size, MXF aspect ratio, MPEG sequence header / VUI); Interpret Footage's + `Interpretation::par` overrides it (`clip.interpretFootage {pixelAspect}`), and + `SequenceSettings::par` is the sequence's. Both go through `project::checked_par` (a zero term + or more than 8:1 either way = square). `render::motion_matrix` works in square display units, + so a 1440 x 1080 clip with 4:3 pixels fills a 1920 x 1080 square-pixel sequence at 100 % + (Scale to Frame Size, Set to Frame Size and Fit / Fill use `Project::conformed_source_size`); + graphics and adjustment layers are in sequence pixels. New Sequence From Clip copies the clip's + ratio, and the Source and Program monitors draw the picture at its display aspect. The + renderer does not read decoded frames' own `VideoFrame::par`. - **Offline media and proxies.** The pool caches each item's source per reference (path, offline flag), so relinking and undo take effect at once. Media that can't be opened renders the offline slate (`render::offline`) instead of failing. With proxies enabled, an item with a proxy reads diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 7bca6de06..a860e269e 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -118,7 +118,7 @@ no media outside its range, so there a kept In is moved to the subclip's first o the result's `sourceInClamped: [ids]` (present with `keepSourceIn`) names the clips it was moved for. **Colour** (`crates/engine/src/color.rs`): `sequence.colorSettings {workingSpace: rec709|rec2100-pq| -rec2100-hlg, wideGamut, autoToneMap}`, `clip.interpretFootage {items?, colorSpace: auto|}`, +rec2100-hlg, wideGamut, autoToneMap}`, `clip.interpretFootage {items?, colorSpace?: auto|, pixelAspect?: [num, den]|file}`, `color.spaces`, `media.colorInfo {item}`; LUTs: `lut.import {path, name?}`, `lut.list`, `lut.remove {id}`, `lut.export {lut, path, format?}`, `lumetri.setInputLut` / `lumetri.setLook {clip?, lut: lib:|builtin:|"", path?}`, `lumetri.setSection {clip?, section, on?}`,