From d1c1e23bb9064afe77e2b1f098dbc243a544e0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Kozlovsk=C3=BD?= Date: Fri, 16 Oct 2026 19:29:35 +0200 Subject: [PATCH] xtask bench --repeat N: median/p90/min/max summary, compare by median Runs every scenario N times (default 1, unchanged). Every run is still appended to results.jsonl; with N > 1 one "kind": "summary" row per key follows, whose metrics are the medians of the successful runs (user_s and sys_s included, byte/instruction counts rounded) and whose summary holds median, p90 (nearest rank, = max for N < 10), min and max. A failed run marks the summary failed so the gate refuses it. --compare uses the latest row per key; the table labels each side "median of N" or "1 run" and warns when they differ, and --fail-over refuses such keys. --repeat with --with is rejected. docs/testing.md documents the format. --- docs/testing.md | 9 ++ xtask/src/bench.rs | 241 ++++++++++++++++++++++++++++++++++++++++--- xtask/tests/bench.rs | 28 +++++ 3 files changed, 266 insertions(+), 12 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 9017e0c2..3a975747 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -68,6 +68,15 @@ instructions retired; `/usr/bin/time -v` on Linux gives max RSS and CPU; elsewhe It prints a table and appends one JSON line per run to `target/bench/results.jsonl` (git rev, date, load average, scenario, settings, metrics); `--compare OLD.jsonl` prints the % change against the latest matching runs there, `--compare OLD --with NEW` compares two files without rendering. +`--repeat N` runs every scenario N times; with N > 1, one `"kind": "summary"` line per key follows +the runs. Its `metrics` are the medians of the successful runs (byte and instruction counts +rounded), `summary` holds `runs`, `failed` and median/p90/min/max per metric (p90 is the nearest +rank, so it equals the max for N < 10), and its other fields (`date`, `load`, ...) are those of the +key's last run; scripts that average runs should skip `kind: summary` lines. One failed run marks +the summary failed. Comparisons use the latest line per key, so a key is a median only if its +summary is the latest line (a later single run, or a batch that aborted before its summaries, +leaves a raw run latest). The table labels each side `median of N` or `1 run` and warns when they +differ; `--fail-over` refuses such keys. The table notes the 1-minute load average: on a busy machine wall time is noise, so compare peak memory and instructions retired, and rerun when the load differs a lot. diff --git a/xtask/src/bench.rs b/xtask/src/bench.rs index fdfc3443..b368d942 100644 --- a/xtask/src/bench.rs +++ b/xtask/src/bench.rs @@ -4,7 +4,8 @@ //! commands, no committed media), renders each at Full and Half plus one single frame, and measures //! every run with `/usr/bin/time` (macOS `-l`: peak memory footprint, max RSS, instructions //! retired; Linux `-v`: max RSS; elsewhere wall time only). Prints a table and appends one JSON -//! line per run to `target/bench/results.jsonl`; `--compare OLD.jsonl` prints the % change. +//! line per run to `target/bench/results.jsonl`; with `--repeat N` (N > 1) one `"kind": "summary"` +//! line per key follows, whose metrics are the medians. `--compare OLD.jsonl` prints the % change. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -28,7 +29,14 @@ Run it under `nice -n 10` on a busy machine; the table notes the load average of --quick fewer frames, for a fast check --format F render format (default png: a numbered sequence, deleted after each run) --regen regenerate the footage media (otherwise reused when present) - --compare OLD.jsonl print the % change of this run against the latest matching runs in OLD + --repeat N run every scenario N times (default 1). Every run is appended as before, + followed by one `kind: summary` row per key whose metrics are the + medians (`summary` holds median, p90, min and max; p90 is the nearest + rank, so it equals max for N < 10). Not allowed with --with + --compare OLD.jsonl print the % change of this run against the latest matching runs in OLD. + The latest row per key is compared, so a side is a median only when its + summary is the latest row; each side is labelled `median of N` or `1 run` + and mixing them is warned about (refused with --fail-over) --with NEW.jsonl with --compare: compare two result files, without rendering --fail-over PCT with --compare: exit 1 when a matched run got more than PCT % worse in CPU time, peak memory, max RSS or instructions retired (wall time is @@ -57,10 +65,11 @@ struct Opts { compare: Option, with: Option, fail_over: Option, + repeat: u32, } fn parse_opts(args: &[String]) -> Result { - let mut o = Opts { format: "png".into(), ..Default::default() }; + let mut o = Opts { format: "png".into(), repeat: 1, ..Default::default() }; let mut it = args.iter(); while let Some(a) = it.next() { let mut val = |name: &str| it.next().cloned().ok_or_else(|| format!("{name} needs a value")); @@ -74,6 +83,7 @@ fn parse_opts(args: &[String]) -> Result { "--quick" => o.quick = true, "--format" => o.format = val("--format")?, "--regen" => o.regen = true, + "--repeat" => o.repeat = val("--repeat")?.parse().ok().filter(|n| *n >= 1).ok_or("--repeat: a number of at least 1 expected")?, "--compare" => o.compare = Some(PathBuf::from(val("--compare")?)), "--with" => o.with = Some(PathBuf::from(val("--with")?)), "--fail-over" => { @@ -88,6 +98,9 @@ fn parse_opts(args: &[String]) -> Result { if o.with.is_some() && o.compare.is_none() { return Err("--with needs --compare OLD.jsonl".into()); } + if o.with.is_some() && args.iter().any(|a| a == "--repeat") { + return Err("--repeat has no effect with --with (nothing is rendered)".into()); + } if o.fail_over.is_some() && o.compare.is_none() { return Err("--fail-over needs --compare OLD.jsonl".into()); } @@ -134,17 +147,30 @@ pub fn run(root: &Path, args: &[String]) -> Result<(), String> { let ctx = RunContext::new(root, &bin, &o); let results_path = dir.join("results.jsonl"); let mut rows = Vec::new(); - for sc in &scenarios { - for kind in RUNS { - eprintln!("bench: {} {kind}", sc.name); - let row = measure(&bin, &dir, sc, kind, frames, &o.format, timer, &ctx)?; - append_line(&results_path, &row)?; - rows.push(row); + for i in 1..=o.repeat { + for sc in &scenarios { + for kind in RUNS { + if o.repeat > 1 { + eprintln!("bench: {} {kind} ({i}/{})", sc.name, o.repeat); + } else { + eprintln!("bench: {} {kind}", sc.name); + } + let row = measure(&bin, &dir, sc, kind, frames, &o.format, timer, &ctx)?; + append_line(&results_path, &row)?; + rows.push(row); + } } } print!("{}", table(&rows, timer)); - println!("appended {} runs to {}", rows.len(), results_path.display()); + let summaries = if o.repeat > 1 { summarize(&rows) } else { vec![] }; + for row in &summaries { + append_line(&results_path, row)?; + } + print!("{}", summary_table(&summaries)); + println!("appended {} runs and {} summary rows to {}", rows.len(), summaries.len(), results_path.display()); if let Some(old) = &o.compare { + // Summaries come after their runs, so the latest row per key is the median when present. + rows.extend(summaries); let (old, new) = (latest_per_key(&read_jsonl(old)?), latest_per_key(&rows)); print!("{}", compare_table(&old, &new)); return check_regressions(&old, &new, o.fail_over); @@ -842,6 +868,88 @@ fn table(rows: &[Value], timer: TimeTool) -> String { s } +/// Median, p90 (nearest rank), min and max of `v`; `None` when empty. +fn stats(v: &[f64]) -> Option<[f64; 4]> { + let mut v: Vec = v.iter().copied().filter(|x| x.is_finite()).collect(); + if v.is_empty() { + return None; + } + v.sort_by(f64::total_cmp); + let n = v.len(); + let median = if n % 2 == 1 { v[n / 2] } else { (v[n / 2 - 1] + v[n / 2]) / 2.0 }; + let p90 = v[(n * 9).div_ceil(10).max(1) - 1]; + Some([median, p90, v[0], v[n - 1]]) +} + +const STATS: [&str; 4] = ["median", "p90", "min", "max"]; + +/// One `"kind": "summary"` row per key over repeated runs: the key fields (and `date`, `load`, +/// ...) of its latest run, `metrics` set to the medians of the successful runs (byte and +/// instruction counts rounded), `summary` with every statistic, and `ok` only when every +/// run succeeded (so `--fail-over` still refuses a key where a run failed). +fn summarize(rows: &[Value]) -> Vec { + let mut groups: BTreeMap> = BTreeMap::new(); + for r in rows { + groups.entry(key_of(r)).or_default().push(r); + } + let mut out = vec![]; + for runs in groups.values() { + let Some(last) = runs.last() else { continue }; + let good: Vec<&Value> = runs.iter().copied().filter(|r| r["ok"] == true).collect(); + let failed = runs.len() - good.len(); + let mut row = (*last).clone(); + let mut metrics = serde_json::Map::new(); + let mut summary = serde_json::Map::new(); + for field in COMPARED.iter().map(|(f, _)| *f).chain(["user_s", "sys_s"]) { + let values: Vec = good.iter().filter_map(|r| r["metrics"][field].as_f64()).collect(); + let st = stats(&values); + let integer = field.ends_with("_bytes") || field == "instructions"; + let median = |s: [f64; 4]| if integer { serde_json::json!(s[0].round() as u64) } else { serde_json::json!(s[0]) }; + metrics.insert(field.to_string(), st.map_or(Value::Null, median)); + summary.insert(field.to_string(), st.map_or(Value::Null, |s| STATS.iter().zip(s).map(|(k, v)| (k.to_string(), serde_json::json!(v))).collect())); + } + if metrics["wall_s"].is_null() { + metrics.insert("wall_s".into(), serde_json::json!(0.0)); + } + row["kind"] = Value::String("summary".into()); + row["metrics"] = Value::Object(metrics); + row["summary"] = serde_json::json!({"runs": runs.len(), "failed": failed, "stats": summary}); + row["ok"] = Value::Bool(failed == 0); + if failed > 0 { + row["error"] = Value::String(format!("{failed} of {} runs failed", runs.len())); + } + out.push(row); + } + out +} + +fn summary_table(summaries: &[Value]) -> String { + let Some(first) = summaries.first() else { return String::new() }; + let mut s = format!("\nsummary over {} runs per key\n", first["summary"]["runs"]); + s.push_str(&format!("{:<22} {:<6} {:<6} {:>8} {:>8} {:>9} {:>9} {:>8}\n", "scenario", "run", "stat", "wall s", "cpu s", "peak MB", "maxRSS MB", "Ginstr")); + for r in summaries { + let st = &r["summary"]["stats"]; + for stat in STATS { + let ginstr = st["instructions"][stat].as_f64().map(|i| format!("{:.1}", i / 1e9)).unwrap_or_else(|| "-".into()); + s.push_str(&format!( + "{:<22} {:<6} {:<6} {:>8} {:>8} {:>9} {:>9} {:>8}\n", + clip(r["scenario"].as_str().unwrap_or(""), 22), + r["run"].as_str().unwrap_or(""), + stat, + num(&st["wall_s"][stat], 2), + num(&st["cpu_s"][stat], 2), + mb(&st["peak_footprint_bytes"][stat]), + mb(&st["max_rss_bytes"][stat]), + ginstr, + )); + } + if r["ok"] != true { + s.push_str(&format!(" ({})\n", r["error"].as_str().unwrap_or(""))); + } + } + s +} + fn clip(s: &str, n: usize) -> String { if s.chars().count() <= n { s.to_string() } else { s.chars().take(n.saturating_sub(1)).chain(['…']).collect() } } @@ -913,6 +1021,9 @@ fn check_regressions(old: &BTreeMap, new: &BTreeMap, new: &BTreeMap bool { + row["kind"] == "summary" +} + +/// `median of N` for a summary row, `1 run` otherwise. +fn sample(row: &Value) -> String { + if is_summary(row) { format!("median of {}", row["summary"]["runs"]) } else { "1 run".into() } +} + fn compare_table(old: &BTreeMap, new: &BTreeMap) -> String { let mut s = String::from("\n% change against the earlier run (negative = better; instructions retired are the least load-sensitive)\n"); s.push_str(&format!("{:<22} {:<6} {:>6}", "scenario", "run", "frames")); for (_, label) in COMPARED { s.push_str(&format!(" {label:>8}")); } - s.push_str(&format!(" {:>11}\n", "load1 o→n")); + s.push_str(&format!(" {:>11} {}\n", "load1 o→n", "sample o→n")); let mut matched = 0; + let mut mixed = 0; for (k, n) in new { let Some(o) = old.get(k) else { continue }; matched += 1; @@ -958,7 +1079,11 @@ fn compare_table(old: &BTreeMap, new: &BTreeMap) - for (field, _) in COMPARED { s.push_str(&format!(" {:>8}", fmt_pct(pct(o["metrics"][field].as_f64(), n["metrics"][field].as_f64())))); } - s.push_str(&format!(" {:>11}\n", format!("{}→{}", num(&o["load"][0], 0), num(&n["load"][0], 0)))); + mixed += usize::from(is_summary(o) != is_summary(n)); + s.push_str(&format!(" {:>11} {}→{}\n", format!("{}→{}", num(&o["load"][0], 0), num(&n["load"][0], 0)), sample(o), sample(n))); + } + if mixed > 0 { + s.push_str(&format!("warning: {mixed} keys compare a median against a single run\n")); } if matched == 0 { s.push_str("(no matching runs: compare the same scenarios, frames and format)\n"); @@ -1025,6 +1150,98 @@ mod tests { "metrics": {"wall_s": wall, "cpu_s": wall * 2.0, "peak_footprint_bytes": peak, "max_rss_bytes": 0, "instructions": null}}) } + fn five_runs() -> Vec { + [10.0, 30.0, 11.0, 12.0, 9.0].map(|w| row("demo", "full", w, Some(w as u64 * 100 + 1), true)).to_vec() + } + + #[test] + fn stats_median_p90() { + assert_eq!(stats(&[]), None); + assert_eq!(stats(&[3.0]), Some([3.0, 3.0, 3.0, 3.0])); + assert_eq!(stats(&[4.0, 1.0, 3.0, 2.0]), Some([2.5, 4.0, 1.0, 4.0])); + let ten: Vec = (1..=10).map(f64::from).collect(); + assert_eq!(stats(&ten), Some([5.5, 9.0, 1.0, 10.0])); + } + + #[test] + fn stats_small_n_p90_is_max() { + // N = 2: the median is the mean, so one outlier moves it by half; p90 is the max. + assert_eq!(stats(&[1.0, 9.0]), Some([5.0, 9.0, 1.0, 9.0])); + // N = 3: the median ignores the outlier; p90 is still the max. + assert_eq!(stats(&[1.0, 9.0, 1.1]), Some([1.1, 9.0, 1.0, 9.0])); + } + + #[test] + fn summary_row_is_valid_and_median() { + let mut rows = five_runs(); + rows.push(row("demo", "half", 5.0, None, true)); + let sums = summarize(&rows); + assert_eq!(sums.len(), 2); + let full = sums.iter().find(|r| r["run"] == "full").unwrap(); + assert_eq!(full["kind"], "summary"); + assert_eq!(full["metrics"]["wall_s"], 11.0, "median, not the 30 s outlier"); + assert_eq!(full["summary"]["stats"]["wall_s"]["p90"], 30.0); + assert_eq!(full["summary"]["stats"]["wall_s"]["min"], 9.0); + assert_eq!(full["summary"]["runs"], 5); + assert_eq!(full["ok"], true); + assert!(validate_row(full).is_ok()); + assert!(summary_table(&sums).contains("median")); + } + + #[test] + fn summary_keeps_user_sys_and_rounds_integers() { + let mut runs = five_runs()[..2].to_vec(); // peaks 1001 and 3001: even N + for (r, (u, sy)) in runs.iter_mut().zip([(1.0, 0.5), (3.0, 1.5)]) { + r["metrics"]["user_s"] = json!(u); + r["metrics"]["sys_s"] = json!(sy); + } + let m = &summarize(&runs)[0]["metrics"]; + assert_eq!((m["user_s"].clone(), m["sys_s"].clone()), (json!(2.0), json!(1.0))); + assert_eq!(m["peak_footprint_bytes"], json!(2001)); + assert!(m["peak_footprint_bytes"].is_u64()); + } + + #[test] + fn failed_run_fails_summary() { + let mut bad = five_runs(); + bad[1]["ok"] = json!(false); + let s = summarize(&bad); + assert_eq!((s[0]["ok"].clone(), s[0]["metrics"]["wall_s"].clone()), (json!(false), json!(10.5))); + } + + #[test] + fn repeat_parse() { + let a = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + assert_eq!(parse_opts(&[]).unwrap().repeat, 1); + assert_eq!(parse_opts(&a(&["--repeat", "5"])).unwrap().repeat, 5); + assert!(parse_opts(&a(&["--repeat", "0"])).is_err()); + assert!(parse_opts(&a(&["--repeat", "3", "--compare", "a", "--with", "b"])).is_err()); + assert!(parse_opts(&a(&["--repeat", "3", "--compare", "a"])).is_ok()); + } + + #[test] + fn compare_prefers_summary() { + // The summary is appended after its runs, so it is the latest row: median 11 vs 10 → +10 %. + let old = latest_per_key(&[row("demo", "full", 10.0, Some(1001), true)]); + let mut rows = five_runs(); + rows.extend(summarize(&rows)); + let new = latest_per_key(&rows); + assert_eq!(new["demo full full 60 png"]["metrics"]["wall_s"], 11.0); + assert!(regressions(&old, &new, 5.0).iter().any(|r| r.contains("cpu_s +10.0%"))); + } + + #[test] + fn median_against_single_run_is_labelled_and_refused_by_gate() { + let old = latest_per_key(&[row("demo", "full", 10.0, Some(1001), true)]); + let new = latest_per_key(&summarize(&five_runs())); + let table = compare_table(&old, &new); + assert!(table.contains("1 run→median of 5"), "{table}"); + assert!(table.contains("warning: 1 keys compare a median against a single run"), "{table}"); + let e = check_regressions(&old, &new, Some(50.0)).unwrap_err(); + assert!(e.contains("median compared against a single run"), "{e}"); + assert!(check_regressions(&old, &new, None).is_ok(), "exploratory compare still allowed"); + } + #[test] fn compare_maths() { assert_eq!(pct(Some(10.0), Some(8.0)), Some(-20.0)); diff --git a/xtask/tests/bench.rs b/xtask/tests/bench.rs index 9f4a1b2f..919b5b94 100644 --- a/xtask/tests/bench.rs +++ b/xtask/tests/bench.rs @@ -132,3 +132,31 @@ fn valid_results_keep_comparison_and_exploratory_mode() { assert!(output.status.success(), "exploratory comparison stays available"); assert!(diagnostic(&output).contains("failed"), "latest failure is visible: {}", diagnostic(&output)); } + +/// With `--repeat`, a summary row (metrics = medians) follows the runs; comparison uses it. +/// The last raw run (cpu 2.0, +0 %) differs from the median (2.2, +10 %), so only the summary fails +/// the 5 % gate. Both sides are summaries, so the mixed-sample check stays out of the way. +#[test] +fn repeat_summary_rows_are_compared_by_median() { + let dir = ResultsDir::new(); + let summary = |cpu: f64| { + let mut r = row("demo", true); + r["metrics"]["cpu_s"] = json!(cpu); + r["kind"] = json!("summary"); + r["summary"] = json!({"runs": 3, "failed": 0, "stats": {}}); + format!("{r}\n") + }; + let baseline = summary(2.0); + let mut candidate = String::new(); + for wall in [1.1, 9.0, 1.0] { + let mut r = row("demo", true); + r["metrics"]["cpu_s"] = json!(wall * 2.0); + candidate.push_str(&format!("{r}\n")); + } + let raw_only = dir.compare(&baseline, &candidate, false); + assert!(diagnostic(&raw_only).contains("median of 3→1 run"), "{}", diagnostic(&raw_only)); + candidate.push_str(&summary(2.2)); + let output = dir.compare(&baseline, &candidate, true); + assert!(!output.status.success(), "{}", diagnostic(&output)); + assert!(diagnostic(&output).contains("cpu_s +10.0%"), "{}", diagnostic(&output)); +}