mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-24 10:09:01 -06:00
CI: pick build jobs from installed RAM; cap test threads at 4
The fixed four jobs were safe on 8 GB but too many for 4 GB, and slower than needed on large machines. `cargo xtask ci` now uses one build job per 2 GB of installed RAM, capped at the CPU count, and 4 if the RAM can't be read (/proc/meminfo, Win32_ComputerSystem, hw.memsize). With line-table debuginfo, 4 jobs measured about 3 GB of RAM in use and 2 jobs about 1.5 GB. Test threads follow but stop at 4: more barely shortens CI, and at 16 threads on a loaded machine the wall-clock frame-budget test offline_originals_never_block_a_frame failed (one 303 ms frame). Explicit CARGO_BUILD_JOBS / RUST_TEST_THREADS still win. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e9931d6871
commit
6b6e756830
@@ -334,11 +334,12 @@ cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light
|
||||
cargo xtask ci # fmt, clippy, tests, layering, wasm checks
|
||||
```
|
||||
|
||||
CI defaults to line-table debug information, four build jobs and four test
|
||||
threads to avoid many full-debuginfo linkers exhausting RAM. CPU speed and GPU
|
||||
coverage are unchanged. Explicit `CARGO_PROFILE_DEV_DEBUG`, `CARGO_BUILD_JOBS`
|
||||
and `RUST_TEST_THREADS` settings override these defaults (use two jobs/threads
|
||||
on a 4 GB machine). Ordinary development commands keep their existing settings.
|
||||
CI defaults to line-table debug information and one build job per 2 GB of RAM
|
||||
(at most one per CPU; 4 if the RAM can't be read), so an 8 GB machine builds
|
||||
with 4 and a 4 GB machine with 2. Test threads follow, capped at 4. Full-debuginfo linkers, one per CPU,
|
||||
otherwise exhaust RAM before any test runs. GPU coverage is unchanged. Explicit
|
||||
`CARGO_PROFILE_DEV_DEBUG`, `CARGO_BUILD_JOBS` and `RUST_TEST_THREADS` settings
|
||||
override these defaults. Ordinary development commands keep their own settings.
|
||||
|
||||
**Chinese and Japanese text** need the shared font repo, an optional build input (official releases always include it):
|
||||
|
||||
|
||||
+52
-1
@@ -85,7 +85,16 @@ pub fn cargo() -> Command {
|
||||
// tests run. Scope these overridable defaults to CI, including parity/WASM
|
||||
// subprocesses, without changing ordinary developer builds or GPU coverage.
|
||||
if std::env::args().nth(1).as_deref() == Some("ci") {
|
||||
for (key, value) in [("CARGO_PROFILE_DEV_DEBUG", "line-tables-only"), ("CARGO_BUILD_JOBS", "4"), ("RUST_TEST_THREADS", "4")] {
|
||||
static JOBS: std::sync::OnceLock<(String, String)> = std::sync::OnceLock::new();
|
||||
let (build, threads) = JOBS.get_or_init(|| {
|
||||
let jobs = ci_jobs(total_ram_gb(), std::thread::available_parallelism().map_or(4, |n| n.get()));
|
||||
// More test threads barely shorten CI (a few heavy tests dominate) but make the
|
||||
// wall-clock frame-budget tests flaky on a loaded machine.
|
||||
(jobs.to_string(), jobs.min(4).to_string())
|
||||
});
|
||||
for (key, value) in
|
||||
[("CARGO_PROFILE_DEV_DEBUG", "line-tables-only"), ("CARGO_BUILD_JOBS", build.as_str()), ("RUST_TEST_THREADS", threads.as_str())]
|
||||
{
|
||||
if std::env::var_os(key).is_none() {
|
||||
c.env(key, value);
|
||||
}
|
||||
@@ -94,6 +103,32 @@ pub fn cargo() -> Command {
|
||||
c
|
||||
}
|
||||
|
||||
/// CI build jobs: one per 2 GB of RAM, at most one per CPU, 4 when the RAM is unknown. With
|
||||
/// line-table debuginfo, 4 jobs measured about 3 GB of RAM in use and 2 jobs about 1.5 GB, so
|
||||
/// 8 GB machines get 4 and 4 GB machines get 2. Test threads are this, capped at 4.
|
||||
fn ci_jobs(ram_gb: Option<u64>, cpus: usize) -> usize {
|
||||
ram_gb.map_or(4, |gb| usize::try_from(gb / 2).unwrap_or(usize::MAX)).clamp(1, cpus.max(1))
|
||||
}
|
||||
|
||||
/// Installed physical memory in whole GB, if the OS reports it.
|
||||
fn total_ram_gb() -> Option<u64> {
|
||||
let bytes: u64 = if cfg!(target_os = "linux") {
|
||||
let info = std::fs::read_to_string("/proc/meminfo").ok()?;
|
||||
let kb = info.lines().find_map(|l| l.strip_prefix("MemTotal:"))?.trim().trim_end_matches("kB").trim();
|
||||
kb.parse::<u64>().ok()?.checked_mul(1024)?
|
||||
} else {
|
||||
let (program, args): (&str, &[&str]) = if cfg!(windows) {
|
||||
("powershell", &["-NoProfile", "-Command", "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory"])
|
||||
} else {
|
||||
("sysctl", &["-n", "hw.memsize"])
|
||||
};
|
||||
let out = Command::new(program).args(args).output().ok()?;
|
||||
String::from_utf8_lossy(&out.stdout).trim().parse().ok()?
|
||||
};
|
||||
// round to the nearest GB: "8 GB" machines report slightly less
|
||||
Some((bytes + (1 << 29)) >> 30)
|
||||
}
|
||||
|
||||
pub fn run(mut cmd: Command, what: &str) -> Result<(), String> {
|
||||
eprintln!("$ {what}");
|
||||
let status = cmd.status().map_err(|e| format!("{what}: failed to spawn: {e}"))?;
|
||||
@@ -359,3 +394,19 @@ Tests that use a corpus skip cleanly when it is absent.
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod ci_jobs_tests {
|
||||
use super::ci_jobs;
|
||||
|
||||
#[test]
|
||||
fn jobs_follow_ram_and_never_exceed_the_cpus() {
|
||||
assert_eq!(ci_jobs(Some(4), 8), 2);
|
||||
assert_eq!(ci_jobs(Some(8), 8), 4);
|
||||
assert_eq!(ci_jobs(Some(32), 32), 16);
|
||||
assert_eq!(ci_jobs(Some(64), 8), 8, "capped at the CPU count");
|
||||
assert_eq!(ci_jobs(Some(1), 8), 1, "at least one");
|
||||
assert_eq!(ci_jobs(None, 32), 4, "unknown RAM keeps the old default");
|
||||
assert_eq!(ci_jobs(None, 2), 2);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user