aleleba/effectcraft
1
0
Fork 0
mirror of https://github.com/storytold/effectcraft.git synced 2026-10-24 10:09:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1689

Browse Source
This commit is contained in:
Brandon Thomas
2026-10-18 09:35:34 -04:00
parent fc3d228616 d1c1e23bb9
commit 40fe8ef969
3 changed files with 266 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/testing.md
+9
View File
@@ -68,6 +68,15 @@ instructions retired; `/usr/bin/time -v` on Linux gives max RSS and CPU; elsewhe
It prints a table and appends one JSON line per run to `target/bench/results.jsonl` (git rev,
date, load average, scenario, settings, metrics); `--compare OLD.jsonl` prints the % change against
the latest matching runs there, `--compare OLD --with NEW` compares two files without rendering.
`--repeat N` runs every scenario N times; with N > 1, one `"kind": "summary"` line per key follows
the runs. Its `metrics` are the medians of the successful runs (byte and instruction counts
rounded), `summary` holds `runs`, `failed` and median/p90/min/max per metric (p90 is the nearest
rank, so it equals the max for N < 10), and its other fields (`date`, `load`, ...) are those of the
key's last run; scripts that average runs should skip `kind: summary` lines. One failed run marks
the summary failed. Comparisons use the latest line per key, so a key is a median only if its
summary is the latest line (a later single run, or a batch that aborted before its summaries,
leaves a raw run latest). The table labels each side `median of N` or `1 run` and warns when they
differ; `--fail-over` refuses such keys.
The table notes the 1-minute load average: on a busy machine wall time is noise, so compare
peak memory and instructions retired, and rerun when the load differs a lot.
xtask/src/bench.rs
+229 -12
View File
@@ -4,7 +4,8 @@
//! commands, no committed media), renders each at Full and Half plus one single frame, and measures
//! every run with `/usr/bin/time` (macOS `-l`: peak memory footprint, max RSS, instructions
//! retired; Linux `-v`: max RSS; elsewhere wall time only). Prints a table and appends one JSON
//! line per run to `target/bench/results.jsonl`; `--compare OLD.jsonl` prints the % change.
//! line per run to `target/bench/results.jsonl`; with `--repeat N` (N > 1) one `"kind": "summary"`
//! line per key follows, whose metrics are the medians. `--compare OLD.jsonl` prints the % change.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
@@ -28,7 +29,14 @@ Run it under `nice -n 10` on a busy machine; the table notes the load average of
--quick fewer frames, for a fast check
--format F render format (default png: a numbered sequence, deleted after each run)
--regen regenerate the footage media (otherwise reused when present)
--compare OLD.jsonl print the % change of this run against the latest matching runs in OLD
--repeat N run every scenario N times (default 1). Every run is appended as before,
followed by one `kind: summary` row per key whose metrics are the
medians (`summary` holds median, p90, min and max; p90 is the nearest
rank, so it equals max for N < 10). Not allowed with --with
--compare OLD.jsonl print the % change of this run against the latest matching runs in OLD.
The latest row per key is compared, so a side is a median only when its
summary is the latest row; each side is labelled `median of N` or `1 run`
and mixing them is warned about (refused with --fail-over)
--with NEW.jsonl with --compare: compare two result files, without rendering
--fail-over PCT with --compare: exit 1 when a matched run got more than PCT % worse in
CPU time, peak memory, max RSS or instructions retired (wall time is
@@ -57,10 +65,11 @@ struct Opts {
compare: Option<PathBuf>,
with: Option<PathBuf>,
fail_over: Option<f64>,
repeat: u32,
}
fn parse_opts(args: &[String]) -> Result<Opts, String> {
let mut o = Opts { format: "png".into(), ..Default::default() };
let mut o = Opts { format: "png".into(), repeat: 1, ..Default::default() };
let mut it = args.iter();
while let Some(a) = it.next() {
let mut val = |name: &str| it.next().cloned().ok_or_else(|| format!("{name} needs a value"));
@@ -74,6 +83,7 @@ fn parse_opts(args: &[String]) -> Result<Opts, String> {
"--quick" => o.quick = true,
"--format" => o.format = val("--format")?,
"--regen" => o.regen = true,
"--repeat" => o.repeat = val("--repeat")?.parse().ok().filter(|n| *n >= 1).ok_or("--repeat: a number of at least 1 expected")?,
"--compare" => o.compare = Some(PathBuf::from(val("--compare")?)),
"--with" => o.with = Some(PathBuf::from(val("--with")?)),
"--fail-over" => {
@@ -88,6 +98,9 @@ fn parse_opts(args: &[String]) -> Result<Opts, String> {
if o.with.is_some() && o.compare.is_none() {
return Err("--with needs --compare OLD.jsonl".into());
}
if o.with.is_some() && args.iter().any(|a| a == "--repeat") {
return Err("--repeat has no effect with --with (nothing is rendered)".into());
}
if o.fail_over.is_some() && o.compare.is_none() {
return Err("--fail-over needs --compare OLD.jsonl".into());
}
@@ -134,17 +147,30 @@ pub fn run(root: &Path, args: &[String]) -> Result<(), String> {
let ctx = RunContext::new(root, &bin, &o);
let results_path = dir.join("results.jsonl");
let mut rows = Vec::new();
for sc in &scenarios {
for kind in RUNS {
eprintln!("bench: {} {kind}", sc.name);
let row = measure(&bin, &dir, sc, kind, frames, &o.format, timer, &ctx)?;
append_line(&results_path, &row)?;
rows.push(row);
for i in 1..=o.repeat {
for sc in &scenarios {
for kind in RUNS {
if o.repeat > 1 {
eprintln!("bench: {} {kind} ({i}/{})", sc.name, o.repeat);
} else {
eprintln!("bench: {} {kind}", sc.name);
}
let row = measure(&bin, &dir, sc, kind, frames, &o.format, timer, &ctx)?;
append_line(&results_path, &row)?;
rows.push(row);
}
}
}
print!("{}", table(&rows, timer));
println!("appended {} runs to {}", rows.len(), results_path.display());
let summaries = if o.repeat > 1 { summarize(&rows) } else { vec![] };
for row in &summaries {
append_line(&results_path, row)?;
}
print!("{}", summary_table(&summaries));
println!("appended {} runs and {} summary rows to {}", rows.len(), summaries.len(), results_path.display());
if let Some(old) = &o.compare {
// Summaries come after their runs, so the latest row per key is the median when present.
rows.extend(summaries);
let (old, new) = (latest_per_key(&read_jsonl(old)?), latest_per_key(&rows));
print!("{}", compare_table(&old, &new));
return check_regressions(&old, &new, o.fail_over);
@@ -842,6 +868,88 @@ fn table(rows: &[Value], timer: TimeTool) -> String {
s
}
/// Median, p90 (nearest rank), min and max of `v`; `None` when empty.
fn stats(v: &[f64]) -> Option<[f64; 4]> {
let mut v: Vec<f64> = v.iter().copied().filter(|x| x.is_finite()).collect();
if v.is_empty() {
return None;
}
v.sort_by(f64::total_cmp);
let n = v.len();
let median = if n % 2 == 1 { v[n / 2] } else { (v[n / 2 - 1] + v[n / 2]) / 2.0 };
let p90 = v[(n * 9).div_ceil(10).max(1) - 1];
Some([median, p90, v[0], v[n - 1]])
}
const STATS: [&str; 4] = ["median", "p90", "min", "max"];
/// One `"kind": "summary"` row per key over repeated runs: the key fields (and `date`, `load`,
/// ...) of its latest run, `metrics` set to the medians of the successful runs (byte and
/// instruction counts rounded), `summary` with every statistic, and `ok` only when every
/// run succeeded (so `--fail-over` still refuses a key where a run failed).
fn summarize(rows: &[Value]) -> Vec<Value> {
let mut groups: BTreeMap<String, Vec<&Value>> = BTreeMap::new();
for r in rows {
groups.entry(key_of(r)).or_default().push(r);
}
let mut out = vec![];
for runs in groups.values() {
let Some(last) = runs.last() else { continue };
let good: Vec<&Value> = runs.iter().copied().filter(|r| r["ok"] == true).collect();
let failed = runs.len() - good.len();
let mut row = (*last).clone();
let mut metrics = serde_json::Map::new();
let mut summary = serde_json::Map::new();
for field in COMPARED.iter().map(|(f, _)| *f).chain(["user_s", "sys_s"]) {
let values: Vec<f64> = good.iter().filter_map(|r| r["metrics"][field].as_f64()).collect();
let st = stats(&values);
let integer = field.ends_with("_bytes") || field == "instructions";
let median = |s: [f64; 4]| if integer { serde_json::json!(s[0].round() as u64) } else { serde_json::json!(s[0]) };
metrics.insert(field.to_string(), st.map_or(Value::Null, median));
summary.insert(field.to_string(), st.map_or(Value::Null, |s| STATS.iter().zip(s).map(|(k, v)| (k.to_string(), serde_json::json!(v))).collect()));
}
if metrics["wall_s"].is_null() {
metrics.insert("wall_s".into(), serde_json::json!(0.0));
}
row["kind"] = Value::String("summary".into());
row["metrics"] = Value::Object(metrics);
row["summary"] = serde_json::json!({"runs": runs.len(), "failed": failed, "stats": summary});
row["ok"] = Value::Bool(failed == 0);
if failed > 0 {
row["error"] = Value::String(format!("{failed} of {} runs failed", runs.len()));
}
out.push(row);
}
out
}
fn summary_table(summaries: &[Value]) -> String {
let Some(first) = summaries.first() else { return String::new() };
let mut s = format!("\nsummary over {} runs per key\n", first["summary"]["runs"]);
s.push_str(&format!("{:<22} {:<6} {:<6} {:>8} {:>8} {:>9} {:>9} {:>8}\n", "scenario", "run", "stat", "wall s", "cpu s", "peak MB", "maxRSS MB", "Ginstr"));
for r in summaries {
let st = &r["summary"]["stats"];
for stat in STATS {
let ginstr = st["instructions"][stat].as_f64().map(|i| format!("{:.1}", i / 1e9)).unwrap_or_else(|| "-".into());
s.push_str(&format!(
"{:<22} {:<6} {:<6} {:>8} {:>8} {:>9} {:>9} {:>8}\n",
clip(r["scenario"].as_str().unwrap_or(""), 22),
r["run"].as_str().unwrap_or(""),
stat,
num(&st["wall_s"][stat], 2),
num(&st["cpu_s"][stat], 2),
mb(&st["peak_footprint_bytes"][stat]),
mb(&st["max_rss_bytes"][stat]),
ginstr,
));
}
if r["ok"] != true {
s.push_str(&format!(" ({})\n", r["error"].as_str().unwrap_or("")));
}
}
s
}
fn clip(s: &str, n: usize) -> String {
if s.chars().count() <= n { s.to_string() } else { s.chars().take(n.saturating_sub(1)).chain(['…']).collect() }
}
@@ -913,6 +1021,9 @@ fn check_regressions(old: &BTreeMap<String, Value>, new: &BTreeMap<String, Value
missing.push(format!("failed latest {label} run: {key}"));
}
}
if is_summary(baseline) != is_summary(candidate) {
missing.push(format!("median compared against a single run ({} vs {}): {key}", sample(baseline), sample(candidate)));
}
if !GATED.iter().any(|field| pct(baseline["metrics"][*field].as_f64(), candidate["metrics"][*field].as_f64()).is_some()) {
missing.push(format!("no comparable resource measurements: {key}"));
}
@@ -934,14 +1045,24 @@ fn check_regressions(old: &BTreeMap<String, Value>, new: &BTreeMap<String, Value
}
}
fn is_summary(row: &Value) -> bool {
row["kind"] == "summary"
}
/// `median of N` for a summary row, `1 run` otherwise.
fn sample(row: &Value) -> String {
if is_summary(row) { format!("median of {}", row["summary"]["runs"]) } else { "1 run".into() }
}
fn compare_table(old: &BTreeMap<String, Value>, new: &BTreeMap<String, Value>) -> String {
let mut s = String::from("\n% change against the earlier run (negative = better; instructions retired are the least load-sensitive)\n");
s.push_str(&format!("{:<22} {:<6} {:>6}", "scenario", "run", "frames"));
for (_, label) in COMPARED {
s.push_str(&format!(" {label:>8}"));
}
s.push_str(&format!(" {:>11}\n", "load1 o→n"));
s.push_str(&format!(" {:>11} {}\n", "load1 o→n", "sample o→n"));
let mut matched = 0;
let mut mixed = 0;
for (k, n) in new {
let Some(o) = old.get(k) else { continue };
matched += 1;
@@ -958,7 +1079,11 @@ fn compare_table(old: &BTreeMap<String, Value>, new: &BTreeMap<String, Value>) -
for (field, _) in COMPARED {
s.push_str(&format!(" {:>8}", fmt_pct(pct(o["metrics"][field].as_f64(), n["metrics"][field].as_f64()))));
}
s.push_str(&format!(" {:>11}\n", format!("{}→{}", num(&o["load"][0], 0), num(&n["load"][0], 0))));
mixed += usize::from(is_summary(o) != is_summary(n));
s.push_str(&format!(" {:>11} {}→{}\n", format!("{}→{}", num(&o["load"][0], 0), num(&n["load"][0], 0)), sample(o), sample(n)));
}
if mixed > 0 {
s.push_str(&format!("warning: {mixed} keys compare a median against a single run\n"));
}
if matched == 0 {
s.push_str("(no matching runs: compare the same scenarios, frames and format)\n");
@@ -1025,6 +1150,98 @@ mod tests {
"metrics": {"wall_s": wall, "cpu_s": wall * 2.0, "peak_footprint_bytes": peak, "max_rss_bytes": 0, "instructions": null}})
}
fn five_runs() -> Vec<Value> {
[10.0, 30.0, 11.0, 12.0, 9.0].map(|w| row("demo", "full", w, Some(w as u64 * 100 + 1), true)).to_vec()
}
#[test]
fn stats_median_p90() {
assert_eq!(stats(&[]), None);
assert_eq!(stats(&[3.0]), Some([3.0, 3.0, 3.0, 3.0]));
assert_eq!(stats(&[4.0, 1.0, 3.0, 2.0]), Some([2.5, 4.0, 1.0, 4.0]));
let ten: Vec<f64> = (1..=10).map(f64::from).collect();
assert_eq!(stats(&ten), Some([5.5, 9.0, 1.0, 10.0]));
}
#[test]
fn stats_small_n_p90_is_max() {
// N = 2: the median is the mean, so one outlier moves it by half; p90 is the max.
assert_eq!(stats(&[1.0, 9.0]), Some([5.0, 9.0, 1.0, 9.0]));
// N = 3: the median ignores the outlier; p90 is still the max.
assert_eq!(stats(&[1.0, 9.0, 1.1]), Some([1.1, 9.0, 1.0, 9.0]));
}
#[test]
fn summary_row_is_valid_and_median() {
let mut rows = five_runs();
rows.push(row("demo", "half", 5.0, None, true));
let sums = summarize(&rows);
assert_eq!(sums.len(), 2);
let full = sums.iter().find(|r| r["run"] == "full").unwrap();
assert_eq!(full["kind"], "summary");
assert_eq!(full["metrics"]["wall_s"], 11.0, "median, not the 30 s outlier");
assert_eq!(full["summary"]["stats"]["wall_s"]["p90"], 30.0);
assert_eq!(full["summary"]["stats"]["wall_s"]["min"], 9.0);
assert_eq!(full["summary"]["runs"], 5);
assert_eq!(full["ok"], true);
assert!(validate_row(full).is_ok());
assert!(summary_table(&sums).contains("median"));
}
#[test]
fn summary_keeps_user_sys_and_rounds_integers() {
let mut runs = five_runs()[..2].to_vec(); // peaks 1001 and 3001: even N
for (r, (u, sy)) in runs.iter_mut().zip([(1.0, 0.5), (3.0, 1.5)]) {
r["metrics"]["user_s"] = json!(u);
r["metrics"]["sys_s"] = json!(sy);
}
let m = &summarize(&runs)[0]["metrics"];
assert_eq!((m["user_s"].clone(), m["sys_s"].clone()), (json!(2.0), json!(1.0)));
assert_eq!(m["peak_footprint_bytes"], json!(2001));
assert!(m["peak_footprint_bytes"].is_u64());
}
#[test]
fn failed_run_fails_summary() {
let mut bad = five_runs();
bad[1]["ok"] = json!(false);
let s = summarize(&bad);
assert_eq!((s[0]["ok"].clone(), s[0]["metrics"]["wall_s"].clone()), (json!(false), json!(10.5)));
}
#[test]
fn repeat_parse() {
let a = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::<Vec<_>>();
assert_eq!(parse_opts(&[]).unwrap().repeat, 1);
assert_eq!(parse_opts(&a(&["--repeat", "5"])).unwrap().repeat, 5);
assert!(parse_opts(&a(&["--repeat", "0"])).is_err());
assert!(parse_opts(&a(&["--repeat", "3", "--compare", "a", "--with", "b"])).is_err());
assert!(parse_opts(&a(&["--repeat", "3", "--compare", "a"])).is_ok());
}
#[test]
fn compare_prefers_summary() {
// The summary is appended after its runs, so it is the latest row: median 11 vs 10 → +10 %.
let old = latest_per_key(&[row("demo", "full", 10.0, Some(1001), true)]);
let mut rows = five_runs();
rows.extend(summarize(&rows));
let new = latest_per_key(&rows);
assert_eq!(new["demo full full 60 png"]["metrics"]["wall_s"], 11.0);
assert!(regressions(&old, &new, 5.0).iter().any(|r| r.contains("cpu_s +10.0%")));
}
#[test]
fn median_against_single_run_is_labelled_and_refused_by_gate() {
let old = latest_per_key(&[row("demo", "full", 10.0, Some(1001), true)]);
let new = latest_per_key(&summarize(&five_runs()));
let table = compare_table(&old, &new);
assert!(table.contains("1 run→median of 5"), "{table}");
assert!(table.contains("warning: 1 keys compare a median against a single run"), "{table}");
let e = check_regressions(&old, &new, Some(50.0)).unwrap_err();
assert!(e.contains("median compared against a single run"), "{e}");
assert!(check_regressions(&old, &new, None).is_ok(), "exploratory compare still allowed");
}
#[test]
fn compare_maths() {
assert_eq!(pct(Some(10.0), Some(8.0)), Some(-20.0));
xtask/tests/bench.rs
+28
View File
@@ -132,3 +132,31 @@ fn valid_results_keep_comparison_and_exploratory_mode() {
assert!(output.status.success(), "exploratory comparison stays available");
assert!(diagnostic(&output).contains("failed"), "latest failure is visible: {}", diagnostic(&output));
}
/// With `--repeat`, a summary row (metrics = medians) follows the runs; comparison uses it.
/// The last raw run (cpu 2.0, +0 %) differs from the median (2.2, +10 %), so only the summary fails
/// the 5 % gate. Both sides are summaries, so the mixed-sample check stays out of the way.
#[test]
fn repeat_summary_rows_are_compared_by_median() {
let dir = ResultsDir::new();
let summary = |cpu: f64| {
let mut r = row("demo", true);
r["metrics"]["cpu_s"] = json!(cpu);
r["kind"] = json!("summary");
r["summary"] = json!({"runs": 3, "failed": 0, "stats": {}});
format!("{r}\n")
};
let baseline = summary(2.0);
let mut candidate = String::new();
for wall in [1.1, 9.0, 1.0] {
let mut r = row("demo", true);
r["metrics"]["cpu_s"] = json!(wall * 2.0);
candidate.push_str(&format!("{r}\n"));
}
let raw_only = dir.compare(&baseline, &candidate, false);
assert!(diagnostic(&raw_only).contains("median of 3→1 run"), "{}", diagnostic(&raw_only));
candidate.push_str(&summary(2.2));
let output = dir.compare(&baseline, &candidate, true);
assert!(!output.status.success(), "{}", diagnostic(&output));
assert!(diagnostic(&output).contains("cpu_s +10.0%"), "{}", diagnostic(&output));
}