diff --git a/AGENTS.md b/AGENTS.md index 129b38b983..a1396fd0e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,6 +124,12 @@ of it. with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. - Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`. +- **Flaky tests:** a test that fails, then passes on a rerun of the same commit, is flaky. Don't rerun until it + passes and move on: open an issue with the failing log, and if it failed on two unrelated PRs, quarantine it with + `#[ignore = "flaky: #"]` (a doctest: an `ignore` code fence and a comment naming the issue) in a PR of its + own until the issue is fixed. Never quarantine a test that fails every time; fix the code or the test. A test that + hangs is a failure too: under `cargo test`, libtest's "has been running for over 60 seconds" line names it; under + nextest (`LIGHTCRAFT_NEXTEST=1`), its `SLOW` lines do. - A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")` before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR). diff --git a/CLAUDE.md b/CLAUDE.md index 83d0b42468..209c73159f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,6 +138,12 @@ of it. with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. - Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`. +- **Flaky tests:** a test that fails, then passes on a rerun of the same commit, is flaky. Don't rerun until it + passes and move on: open an issue with the failing log, and if it failed on two unrelated PRs, quarantine it with + `#[ignore = "flaky: #"]` (a doctest: an `ignore` code fence and a comment naming the issue) in a PR of its + own until the issue is fixed. Never quarantine a test that fails every time; fix the code or the test. A test that + hangs is a failure too: under `cargo test`, libtest's "has been running for over 60 seconds" line names it; under + nextest (`LIGHTCRAFT_NEXTEST=1`), its `SLOW` lines do. - A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")` before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR). diff --git a/xtask/src/ci.rs b/xtask/src/ci.rs index bc3a9b706a..6b55595706 100644 --- a/xtask/src/ci.rs +++ b/xtask/src/ci.rs @@ -92,19 +92,20 @@ fn nextest() -> bool { } /// A test run of `args` (packages, features, targets, filters): `cargo nextest run`, or `cargo -/// test`. The command and how to print it. +/// test`. The command and how to print it. `--no-fail-fast`: a failing test binary doesn't stop +/// the others, so one CI run shows every failure, not just the first crate's. fn tests(args: &[&str]) -> (Command, String) { let mut c = cargo(); let tool = if nextest() { "nextest run" } else { "test" }; - c.args(tool.split(' ')).args(args); - (c, format!("cargo {tool} {}", args.join(" "))) + c.args(tool.split(' ')).arg("--no-fail-fast").args(args); + (c, format!("cargo {tool} --no-fail-fast {}", args.join(" "))) } /// The workspace's doctests. fn doctests() -> (Command, String) { let mut c = cargo(); - c.args(["test", "--workspace", "--doc"]); - (c, "cargo test --workspace --doc".to_owned()) + c.args(["test", "--no-fail-fast", "--workspace", "--doc"]); + (c, "cargo test --no-fail-fast --workspace --doc".to_owned()) } /// `ci --quick`'s tests: build every workspace test target, then run only the test binaries of