S2.53: bench --roto takes the user's own clips and reports peak memory

--clip FRAMES_DIR --truth MASKS_DIR runs the trackers on image sequences with true
mattes (strokes made from the truth; nothing is bundled or copied), tested end to end.
Trackers report the most memory their banks held (peakMB column).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

apps/effectcraft-cli/src/main.rs

@@ -27,7 +27,7 @@
 //!     (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
 //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N]   everyday-operation timings
 //!     on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
-//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--json]   Roto Brush trackers on synthetic clips with known mattes
+//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--clip FRAMES --truth MASKS] [--json]   Roto Brush trackers on synthetic clips with known mattes
 //!     (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)
 //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE    run an After Effects-style script
 //! effectcraft-cli mcp [--autosave | --bridge PORT]            MCP server on stdio
@@ -126,7 +126,7 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
   bench --ops [--small] [--layers N] [--comps N] [--footage N]
                                            everyday operations on a large generated project: startup, open,
                                            save, auto-save, edits + undo/redo, timeline, Project panel
-  bench --roto [--frames N] [--sam2 FILE] [--mobilesam FILE] [--fixes] [--json]
+  bench --roto [--frames N] [--sam2 FILE] [--mobilesam FILE] [--fixes] [--clip FRAMES_DIR --truth MASKS_DIR] [--json]
                                            Roto Brush trackers on synthetic clips with known mattes: movement,
                                            lighting change, occlusion, leaving and re-entering the frame
   script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
@@ -142,6 +142,8 @@ options: --project F.ecproj | --demo | --empty   --save | --save-as F   --bridge
 /// Options that take a value.
 const VALUED: &[&str] = &[
     "--n",
+    "--clip",
+    "--truth",
     "--frames",
     "--sam2",
     "--mobilesam",
@@ -1038,7 +1040,14 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
     let fixes = args.flag("--fixes");
     let measure = |sc: &bench::Scenario, run: &dyn Fn(&bench::Scenario) -> Option<bench::Run>| if fixes { bench::with_fixes(sc, 0.9, 6, run) } else { run(sc) };
     let mut runs = vec![];
-    for sc in bench::scenarios(frames) {
+    let mut scenarios = bench::scenarios(frames);
+    // `--clip FRAMES_DIR --truth MASKS_DIR`: the user's own clip (image sequences, same file order) with its true mattes.
+    if let Some(dir) = args.opt("--clip") {
+        let truth = args.opt("--truth").ok_or_else(|| Failure::Error("--clip needs --truth MASKS_DIR".into()))?;
+        let name = std::path::Path::new(dir).file_name().map_or_else(|| "clip".to_string(), |n| n.to_string_lossy().chars().take(10).collect());
+        scenarios = vec![load_bench_clip(&name, dir, truth, frames).map_err(Failure::Error)?];
+    }
+    for sc in scenarios {
         runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "classical", None))));
         if let Some(m) = &mobilesam {
             runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref())))));
@@ -1055,6 +1064,39 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
     Ok(())
 }
 
+/// The image files of `dir` (png, jpg, jpeg), in file-name order.
+fn image_files(dir: &str) -> Result<Vec<std::path::PathBuf>, String> {
+    let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(dir)
+        .map_err(|e| format!("cannot read {dir}: {e}"))?
+        .filter_map(|e| e.ok().map(|e| e.path()))
+        .filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| ["png", "jpg", "jpeg"].contains(&e.to_ascii_lowercase().as_str())))
+        .collect();
+    files.sort();
+    Ok(files)
+}
+
+/// A benchmark scenario from the user's frames and truth masks (at most `limit` frames, frames at most 480 px
+/// wide; a mask pixel above 127 is the object). Nothing is copied or kept.
+fn load_bench_clip(name: &str, frames_dir: &str, truth_dir: &str, limit: usize) -> Result<effectcraft_track::roto::bench::Scenario, String> {
+    let (frames, masks) = (image_files(frames_dir)?, image_files(truth_dir)?);
+    if frames.is_empty() || frames.len() != masks.len() {
+        return Err(format!("{} frames in {frames_dir} but {} masks in {truth_dir}: they must match one to one", frames.len(), masks.len()));
+    }
+    let n = frames.len().min(limit.clamp(1, 240));
+    let (mut imgs, mut truth) = (vec![], vec![]);
+    for (f, m) in frames.iter().zip(&masks).take(n) {
+        let rgba = image::open(f).map_err(|e| format!("{}: {e}", f.display()))?.to_rgba8();
+        let scale = (480.0 / f64::from(rgba.width().max(1))).min(1.0);
+        let (w, h) = (((f64::from(rgba.width()) * scale).round() as u32).max(1), ((f64::from(rgba.height()) * scale).round() as u32).max(1));
+        let rgba = if (w, h) == rgba.dimensions() { rgba } else { image::imageops::resize(&rgba, w, h, image::imageops::FilterType::Triangle) };
+        let mask = image::open(m).map_err(|e| format!("{}: {e}", m.display()))?.to_luma8();
+        let mask = image::imageops::resize(&mask, w, h, image::imageops::FilterType::Nearest);
+        imgs.push(effectcraft_render::Image::from_rgba8(w, h, rgba.as_raw()));
+        truth.push(mask.as_raw().iter().map(|v| u8::from(*v > 127)).collect());
+    }
+    effectcraft_track::roto::bench::scenario_from(name, imgs, truth)
+}
+
 /// `bench --ops`: everyday operations on a large generated project (200 comps, 5,000 layers in
 /// the main comp, 300 footage items, 20-deep precomp nesting, expressions): startup to first
 /// frame, open, save, auto-save, small and large edits with undo/redo, first frame, timeline

apps/effectcraft-cli/tests/it/cli.rs

@@ -805,3 +805,44 @@ fn render_without_progress_writes_no_progress_lines() {
     let stderr = render_demo_stderr("no-progress.gif", &[]);
     assert!(!stderr.lines().any(|l| l.starts_with("progress")), "{stderr}");
 }
+
+/// `bench --roto --clip FRAMES --truth MASKS` runs the trackers on the user's own image sequences: here a bright disc
+/// drifting over a dark textured frame, with its true masks. Mismatched sequences are refused with a message.
+#[test]
+fn bench_roto_runs_on_a_users_clip() {
+    let (frames, masks) = (tmp("clip-frames"), tmp("clip-masks"));
+    for d in [&frames, &masks] {
+        let _ = std::fs::remove_dir_all(d);
+        std::fs::create_dir_all(d).unwrap();
+    }
+    let (w, h) = (96u32, 72u32);
+    for f in 0..6u32 {
+        let (cx, cy) = (30.0 + 4.0 * f as f32, 36.0f32);
+        let (mut img, mut mask) = (image::RgbImage::new(w, h), image::GrayImage::new(w, h));
+        for y in 0..h {
+            for x in 0..w {
+                let inside = ((x as f32 - cx).powi(2) + (y as f32 - cy).powi(2)).sqrt() < 14.0;
+                let tex = 40 + ((x * 7 + y * 13) % 30) as u8;
+                img.put_pixel(x, y, if inside { image::Rgb([230, 120, 40 + (x % 20) as u8]) } else { image::Rgb([tex / 2, tex, tex + 20]) });
+                mask.put_pixel(x, y, image::Luma([if inside { 255 } else { 0 }]));
+            }
+        }
+        img.save(frames.join(format!("f{f:03}.png"))).unwrap();
+        mask.save(masks.join(format!("f{f:03}.png"))).unwrap();
+    }
+    let (f, m) = (frames.to_str().unwrap(), masks.to_str().unwrap());
+    let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m, "--frames", "6"]).output().unwrap();
+    let text = String::from_utf8_lossy(&out.stdout).to_string();
+    assert!(out.status.success(), "{text}\n{}", String::from_utf8_lossy(&out.stderr));
+    let row = text.lines().find(|l| l.contains("classical")).unwrap_or_else(|| panic!("no classical row: {text}"));
+    let mean: f64 = row.split_whitespace().nth(2).unwrap().parse().unwrap();
+    assert!(mean > 0.8, "classical mean IoU on the user's clip: {mean}\n{text}");
+    // one mask short: refused
+    std::fs::remove_file(masks.join("f005.png")).unwrap();
+    let out = bin().args(["bench", "--roto", "--clip", f, "--truth", m]).output().unwrap();
+    assert!(!out.status.success());
+    assert!(String::from_utf8_lossy(&out.stderr).contains("must match"), "{}", String::from_utf8_lossy(&out.stderr));
+    // --clip without --truth
+    let out = bin().args(["bench", "--roto", "--clip", f]).output().unwrap();
+    assert!(!out.status.success());
+}