diff --git a/apps/deckcraft-cli/src/main.rs b/apps/deckcraft-cli/src/main.rs index fd1d96ea..a7b2fcff 100644 --- a/apps/deckcraft-cli/src/main.rs +++ b/apps/deckcraft-cli/src/main.rs @@ -10,6 +10,7 @@ //! deckcraft-cli describe ID # one command //! deckcraft-cli app [--port PORT] COMMAND [JSON] | --method METHOD [JSON] # drive the running app //! deckcraft-cli mcp [--connect PORT] [--sample] # MCP server over stdio +//! deckcraft-cli bench [FILE | --sample] [--runs N] [--json] # time open, render, save, export //! deckcraft-cli links | --version //! ``` #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -42,6 +43,7 @@ const USAGE: &str = "usage: deckcraft-cli info FILE deckcraft-cli describe COMMAND deckcraft-cli app [--port PORT] COMMAND [JSON] | --method METHOD [JSON] deckcraft-cli mcp [--connect PORT] [--sample] + deckcraft-cli bench [FILE | --sample] [--runs N] [--json] deckcraft-cli links | --version"; fn main() -> ExitCode { @@ -53,6 +55,7 @@ fn main() -> ExitCode { ExitCode::SUCCESS } Some("info") => report(info(rest)), + Some("bench") => report(bench(rest)), Some("render") => report(render(rest)), Some("render-transition") => report(render_transition(rest)), Some("convert") => report(convert(rest)), @@ -105,6 +108,108 @@ fn open(s: &mut Session, input: Option<&str>, sample: bool) -> Result<(), String } } +/// Median and slowest of `runs` timings of `f`, in milliseconds. +fn time_ms(runs: usize, mut f: impl FnMut() -> Result<(), String>) -> Result<(f64, f64), String> { + let mut ms = Vec::with_capacity(runs); + for _ in 0..runs.max(1) { + let t = std::time::Instant::now(); + f()?; + ms.push(t.elapsed().as_secs_f64() * 1000.0); + } + ms.sort_by(f64::total_cmp); + let median = ms.get(ms.len() / 2).copied().unwrap_or(0.0); + Ok((median, ms.last().copied().unwrap_or(0.0))) +} + +/// Peak resident memory of this process in MiB (Linux `VmHWM`); `None` elsewhere. +fn peak_rss_mib() -> Option { + let s = std::fs::read_to_string("/proc/self/status").ok()?; + let kb: f64 = s.lines().find_map(|l| l.strip_prefix("VmHWM:"))?.trim().trim_end_matches("kB").trim().parse().ok()?; + Some(kb / 1024.0) +} + +/// The performance baseline (docs/gaps.md "No baseline"): how long a deck takes to open, to +/// render each slide, to save and to export, and the peak memory, as JSON or a table. +fn bench(args: &[String]) -> Result<(), String> { + use deckcraft_engine::cmd::file::{open_presentation, save_bytes}; + let mut input = None; + let mut sample = false; + let mut runs = 5usize; + let mut as_json = false; + let mut it = args.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "--sample" => sample = true, + "--json" => as_json = true, + "--runs" => { + runs = it.next().and_then(|v| v.parse().ok()).filter(|n| (1..=1000).contains(n)).ok_or("--runs needs a number from 1 to 1000")? + } + f if !f.starts_with("--") && input.is_none() => input = Some(f.to_string()), + o => return Err(format!("bench: unknown argument `{o}`")), + } + } + if input.is_none() && !sample { + return Err("bench needs a FILE or --sample".into()); + } + let mut s = Session::new(); + let (name, bytes) = match &input { + Some(p) => (p.clone(), std::fs::read(p).map_err(|e| format!("{p}: {e}"))?), + None => { + open(&mut s, None, true)?; + let doc = &s.doc().map_err(|e| e.to_string())?.doc; + ("sample.deckcraft".to_string(), save_bytes(doc, "deckcraft").map_err(|e| e.to_string())?) + } + }; + let e = |x: deckcraft_engine::EngineError| x.to_string(); + let doc = open_presentation(&name, &bytes).map_err(e)?; + let slides = doc.slides.len(); + let open = time_ms(runs, || open_presentation(&name, &bytes).map(|_| ()).map_err(e))?; + let opts = deckcraft_engine::render::RenderOpts { scale: 1.0, ..Default::default() }; + let mut per_slide = vec![]; + for i in 0..slides { + per_slide.push(time_ms(runs, || { + deckcraft_engine::render::render_slide(&doc, i, &opts); + Ok(()) + })?); + } + let pptx = save_bytes(&doc, "pptx").map_err(e)?; + let mut steps = vec![ + ("open", open), + ("save .deckcraft", time_ms(runs, || save_bytes(&doc, "deckcraft").map(|_| ()).map_err(e))?), + ("save .pptx", time_ms(runs, || save_bytes(&doc, "pptx").map(|_| ()).map_err(e))?), + ("open .pptx", time_ms(runs, || open_presentation("bench.pptx", &pptx).map(|_| ()).map_err(e))?), + ("export .pdf", time_ms(runs.min(3), || save_bytes(&doc, "pdf").map(|_| ()).map_err(e))?), + ]; + let render_total: f64 = per_slide.iter().map(|m| m.0).sum(); + let render_max = per_slide.iter().map(|m| m.1).fold(0.0, f64::max); + steps.insert(1, ("render all slides (1×)", (render_total, render_max))); + let peak = peak_rss_mib(); + if as_json { + let v = json!({ + "file": name, + "bytes": bytes.len(), + "slides": slides, + "runs": runs, + "steps": steps.iter().map(|(k, (m, x))| json!({"step": k, "medianMs": m, "maxMs": x})).collect::>(), + "renderMsPerSlide": per_slide.iter().map(|m| m.0).collect::>(), + "peakRssMiB": peak, + }); + outln!("{}", serde_json::to_string_pretty(&v).unwrap_or_default()); + } else { + outln!("{name}: {slides} slides, {} KiB, median of {runs} runs", bytes.len() / 1024); + for (k, (m, x)) in &steps { + outln!(" {k:<24} {m:>9.1} ms (slowest {x:.1} ms)"); + } + if let Some((i, m)) = per_slide.iter().enumerate().max_by(|a, b| a.1.0.total_cmp(&b.1.0)) { + outln!(" slowest slide {:>9.1} ms (slide {})", m.0, i + 1); + } + if let Some(p) = peak { + outln!(" peak memory {p:>9.1} MiB"); + } + } + Ok(()) +} + fn info(args: &[String]) -> Result<(), String> { let path = args.first().ok_or("info needs a FILE")?; let mut s = Session::new(); diff --git a/docs/gaps.md b/docs/gaps.md index a2894b42..873165dc 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -236,8 +236,10 @@ Detail in [localization-parity.md](localization-parity.md). ## Performance -- **No baseline.** Nothing measures open time, render time per slide, show frame rate or memory - against PowerPoint. 4–8 h for a benchmark suite in `xtask`. +- **Baseline measured, not compared.** `cargo xtask bench` (and `deckcraft-cli bench FILE`) times + open, per-slide render, save, PPTX round trip and PDF export, and peak memory, against a saved + baseline; show frame rate isn't measured and there are no PowerPoint numbers to set beside it. + 2–4 h. - **Morph aligns outlines every frame** (#67, PR open). 1 h. ## Ecosystem @@ -256,6 +258,7 @@ Detail in [localization-parity.md](localization-parity.md). | Date | Change | Summary | |---|---|---| +| 2026-10-13 | minor | Benchmark suite (`cargo xtask bench`, `deckcraft-cli bench`): the performance gap is now frame rate and a PowerPoint comparison | | 2026-10-13 | minor | Shape, picture and table rendering checked against PowerPoint on ~150 synthetic edge-case slides; 3-D bevels, cameras and face lighting now render approximately (gap narrowed to extrusion, rigs and the 3-D pane) | | 2026-10-12 | minor | IME composition in slide text and notes, and CJK interface text with real glyphs (#101): the IME gap is now Korean on the web and table-cell underlines | | 2026-10-12 | minor | EMF/WMF text, pattern and hatch brushes drawn; legacy OLE and ActiveX previews shown; the SVG gap keeps only EMF clip regions | diff --git a/xtask/src/bench.rs b/xtask/src/bench.rs new file mode 100644 index 00000000..0ee64bd5 --- /dev/null +++ b/xtask/src/bench.rs @@ -0,0 +1,75 @@ +//! `cargo xtask bench`: the performance baseline. Builds `deckcraft-cli` in release, runs its +//! `bench` on the built-in sample (and any decks given), and compares each step with the saved +//! baseline (`docs/benchmarks/baseline.json`) when there is one; `--save` writes a new baseline. + +use std::path::Path; +use std::process::Command; + +use serde_json::{Value, json}; + +/// A step this many times slower than the baseline is flagged. +const SLOWER: f64 = 1.5; + +pub fn run(root: &Path, args: &[&str]) -> Result<(), String> { + let save = args.contains(&"--save"); + let decks: Vec<&str> = args.iter().copied().filter(|a| !a.starts_with("--")).collect(); + let mut build = super::cargo(); + build.args(["build", "--release", "-q", "-p", "deckcraft-cli"]); + super::run(build, "cargo build --release -p deckcraft-cli")?; + let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| root.join("target"), Into::into); + let cli = target.join("release").join(if cfg!(windows) { "deckcraft-cli.exe" } else { "deckcraft-cli" }); + let mut results = vec![]; + let inputs: Vec> = std::iter::once(None).chain(decks.iter().map(|d| Some(*d))).collect(); + for input in inputs { + let mut c = Command::new(&cli); + c.arg("bench").arg("--json"); + match input { + Some(d) => c.arg(d), + None => c.arg("--sample"), + }; + let out = c.output().map_err(|e| format!("{}: {e}", cli.display()))?; + if !out.status.success() { + return Err(format!("bench {}: {}", input.unwrap_or("--sample"), String::from_utf8_lossy(&out.stderr).trim())); + } + let v: Value = serde_json::from_slice(&out.stdout).map_err(|e| format!("bench output: {e}"))?; + results.push(v); + } + let baseline_path = root.join("docs/benchmarks/baseline.json"); + let baseline: Option = std::fs::read(&baseline_path).ok().and_then(|b| serde_json::from_slice(&b).ok()); + let mut slower = 0; + for r in &results { + let file = r["file"].as_str().unwrap_or("?"); + println!("\n{file}: {} slides", r["slides"]); + let base = baseline.as_ref().and_then(|b| b["results"].as_array()?.iter().find(|x| x["file"] == r["file"]).cloned()); + for step in r["steps"].as_array().into_iter().flatten() { + let name = step["step"].as_str().unwrap_or("?"); + let ms = step["medianMs"].as_f64().unwrap_or(0.0); + let was = base.as_ref().and_then(|b| b["steps"].as_array()?.iter().find(|s| s["step"] == step["step"])?["medianMs"].as_f64()); + let cmp = match was { + Some(w) if w > 0.5 => { + let k = ms / w; + if k > SLOWER { + slower += 1; + } + format!(" {k:>5.2}× baseline{}", if k > SLOWER { " SLOWER" } else { "" }) + } + _ => String::new(), + }; + println!(" {name:<24} {ms:>9.1} ms{cmp}"); + } + if let Some(p) = r["peakRssMiB"].as_f64() { + println!(" peak memory {p:>9.1} MiB"); + } + } + if save { + if let Some(dir) = baseline_path.parent() { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let v = json!({"note": "cargo xtask bench --save; times depend on the machine, compare on the same one", "results": results}); + std::fs::write(&baseline_path, serde_json::to_vec_pretty(&v).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; + println!("\nwrote {}", baseline_path.display()); + } else if baseline.is_none() { + println!("\nno baseline yet: `cargo xtask bench --save` writes docs/benchmarks/baseline.json"); + } + if slower > 0 { Err(format!("{slower} step(s) more than {SLOWER}× slower than the baseline")) } else { Ok(()) } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e64a0d42..fefa20bf 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -4,6 +4,7 @@ //! invoked through `std::process::Command`. mod assets; +mod bench; mod ico; mod layers; mod parity; @@ -22,6 +23,9 @@ commands: wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates ci fmt --check, clippy -D warnings, test, assets, layers, wasm (stops at first failure) stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) + bench [--save] [DECK...] + time open, render, save and export of the sample (and DECKs) in release, and + compare with docs/benchmarks/baseline.json (--save writes it) parity recompute the PowerPoint feature-parity summary in docs/parity-checklist.md ico ... pack square PNGs (<= 256 px) into a Windows .ico (see packaging/icons.sh) @@ -37,6 +41,7 @@ fn main() -> ExitCode { Some("ci") => cmd_ci(), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), Some("parity") => parity::run(&root()), + Some("bench") => bench::run(&root(), &rest), Some("ico") => ico::run(&rest), Some("version") => version::run(&root(), &rest), Some("-h" | "--help" | "help") | None => {