mirror of
https://github.com/storytold/lightcraft.git
synced 2026-11-22 14:06:50 -06:00
xtask ci: --no-fail-fast test runs; flaky-test policy (#3791)
* xtask ci: --no-fail-fast test runs; flaky-test policy * Flaky-test policy: doctest quarantine and nextest's SLOW lines
This commit is contained in:
@@ -124,6 +124,12 @@ of it.
|
||||
with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe
|
||||
≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG.
|
||||
- Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`.
|
||||
- **Flaky tests:** a test that fails, then passes on a rerun of the same commit, is flaky. Don't rerun until it
|
||||
passes and move on: open an issue with the failing log, and if it failed on two unrelated PRs, quarantine it with
|
||||
`#[ignore = "flaky: #<issue>"]` (a doctest: an `ignore` code fence and a comment naming the issue) in a PR of its
|
||||
own until the issue is fixed. Never quarantine a test that fails every time; fix the code or the test. A test that
|
||||
hangs is a failure too: under `cargo test`, libtest's "has been running for over 60 seconds" line names it; under
|
||||
nextest (`LIGHTCRAFT_NEXTEST=1`), its `SLOW` lines do.
|
||||
- A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")`
|
||||
before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and
|
||||
data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR).
|
||||
|
||||
@@ -138,6 +138,12 @@ of it.
|
||||
with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe
|
||||
≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG.
|
||||
- Never commit media: test images are procedural (`lightcraft-scenes`) or CC0 downloads in the gitignored `corpus/`.
|
||||
- **Flaky tests:** a test that fails, then passes on a rerun of the same commit, is flaky. Don't rerun until it
|
||||
passes and move on: open an issue with the failing log, and if it failed on two unrelated PRs, quarantine it with
|
||||
`#[ignore = "flaky: #<issue>"]` (a doctest: an `ignore` code fence and a comment naming the issue) in a PR of its
|
||||
own until the issue is fixed. Never quarantine a test that fails every time; fix the code or the test. A test that
|
||||
hangs is a failure too: under `cargo test`, libtest's "has been running for over 60 seconds" line names it; under
|
||||
nextest (`LIGHTCRAFT_NEXTEST=1`), its `SLOW` lines do.
|
||||
- A test that can't run here (no GPU, corpus file, craft-fonts face, model) calls `lightcraft_testkit::skip!(Gpu, "why")`
|
||||
before it returns, not a bare `eprintln!`. `cargo xtask ci` then lists the skips and the machine's GPU adapters and
|
||||
data, and writes `target/ci/report.json` and `target/ci/summary.md` (Markdown for a PR).
|
||||
|
||||
+6
-5
@@ -92,19 +92,20 @@ fn nextest() -> bool {
|
||||
}
|
||||
|
||||
/// A test run of `args` (packages, features, targets, filters): `cargo nextest run`, or `cargo
|
||||
/// test`. The command and how to print it.
|
||||
/// test`. The command and how to print it. `--no-fail-fast`: a failing test binary doesn't stop
|
||||
/// the others, so one CI run shows every failure, not just the first crate's.
|
||||
fn tests(args: &[&str]) -> (Command, String) {
|
||||
let mut c = cargo();
|
||||
let tool = if nextest() { "nextest run" } else { "test" };
|
||||
c.args(tool.split(' ')).args(args);
|
||||
(c, format!("cargo {tool} {}", args.join(" ")))
|
||||
c.args(tool.split(' ')).arg("--no-fail-fast").args(args);
|
||||
(c, format!("cargo {tool} --no-fail-fast {}", args.join(" ")))
|
||||
}
|
||||
|
||||
/// The workspace's doctests.
|
||||
fn doctests() -> (Command, String) {
|
||||
let mut c = cargo();
|
||||
c.args(["test", "--workspace", "--doc"]);
|
||||
(c, "cargo test --workspace --doc".to_owned())
|
||||
c.args(["test", "--no-fail-fast", "--workspace", "--doc"]);
|
||||
(c, "cargo test --no-fail-fast --workspace --doc".to_owned())
|
||||
}
|
||||
|
||||
/// `ci --quick`'s tests: build every workspace test target, then run only the test binaries of
|
||||
|
||||
Reference in New Issue
Block a user