mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-27 03:27:14 -06:00
refactor(lightcraft-engine): move denoise gpu_tests to denoise/gpu_tests.rs
This commit is contained in:
aiongg
@@ -1643,311 +1643,4 @@ impl Session {
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "denoise"))]
|
||||
mod gpu_tests {
|
||||
use super::*;
|
||||
use lightcraft_denoise::manifest::{Domain, Gain};
|
||||
|
||||
/// A folder of the test's own: the card's set-up leaves its marker beside the model.
|
||||
fn folder(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
/// The model file is read again when the card is set up (for the first photo), so it stays until the test ends.
|
||||
fn model(name: &str, tile: u64) -> (Arc<dyn Model>, lightcraft_denoise::runtime::CpuRunner, PathBuf) {
|
||||
let path = folder(name).join("model.onnx");
|
||||
std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap();
|
||||
let manifest = DenoiserManifest {
|
||||
id: "synthetic".into(),
|
||||
name: "Synthetic".into(),
|
||||
version: "1".into(),
|
||||
licence: Default::default(),
|
||||
source: None,
|
||||
sha256: None,
|
||||
size_bytes: None,
|
||||
provenance: String::new(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: tile as u32,
|
||||
overlap: 16,
|
||||
gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },
|
||||
};
|
||||
let loaded = default_loader()(&path, &manifest).unwrap();
|
||||
let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap();
|
||||
(loaded, cpu, path)
|
||||
}
|
||||
|
||||
fn worst(want: &[f32], got: &[f32]) -> f32 {
|
||||
assert_eq!(want.len(), got.len());
|
||||
let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs()));
|
||||
want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tile_on_the_card_is_the_cpus_tile() {
|
||||
let (model, cpu, path) = model("same", 64);
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
let (runner, at_once) = model.runner(RunOn::Gpu, 16);
|
||||
assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3);
|
||||
let device = model.device(RunOn::Gpu);
|
||||
eprintln!("device: {device}");
|
||||
match device["kind"].as_str() {
|
||||
Some("gpu") => {
|
||||
assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty()));
|
||||
assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once");
|
||||
assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}");
|
||||
}
|
||||
Some("cpu") => {
|
||||
assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}");
|
||||
assert_eq!(at_once, 16);
|
||||
}
|
||||
other => panic!("the first photo decides where the work runs, not {other:?}"),
|
||||
}
|
||||
assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
/// A server on this computer running `cpu` for the model with `sha256`.
|
||||
fn server(cpu: lightcraft_denoise::runtime::CpuRunner, sha256: &str, tile: usize) -> String {
|
||||
use lightcraft_denoise::remote::{ModelId, ServerOptions, serve};
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = l.local_addr().unwrap().to_string();
|
||||
let id = ModelId { id: "synthetic".into(), version: "1".into(), sha256: sha256.into() };
|
||||
let run: lightcraft_denoise::remote::RunTile = Arc::new(move |input: &[f32]| cpu.run(input));
|
||||
let opts = ServerOptions { token: "remote-test-token-0123".into(), ..Default::default() };
|
||||
std::thread::spawn(move || serve(l, id, tile, "CPU (test server)".into(), run, opts, Arc::new(|_| {})));
|
||||
addr
|
||||
}
|
||||
|
||||
fn with_remote(path: &Path, tile: u64, addr: &str, sha256: &str) -> Network {
|
||||
let manifest = DenoiserManifest {
|
||||
id: "synthetic".into(),
|
||||
name: "Synthetic".into(),
|
||||
version: "1".into(),
|
||||
licence: Default::default(),
|
||||
source: None,
|
||||
sha256: None,
|
||||
size_bytes: None,
|
||||
provenance: String::new(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: tile as u32,
|
||||
overlap: 16,
|
||||
gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },
|
||||
};
|
||||
let remote = RemoteSide::new(lightcraft_denoise::remote::Remote::new(addr, "remote-test-token-0123", sha256, tile as usize));
|
||||
network(path, &manifest, Some(remote)).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_remote_server_shares_the_tiles_and_drops_out_when_it_cannot() {
|
||||
let (_, cpu, path) = model("remote", 64);
|
||||
let sha = lightcraft_denoise::hash::sha256_file(&path).unwrap();
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
// the same model on the server: its slots come on top of this computer's threads, answers within f16 rounding
|
||||
let addr = server(lightcraft_denoise::runtime::CpuRunner::load(&path, &cpu.manifest().clone()).unwrap(), &sha, 64);
|
||||
let net = with_remote(&path, 64, &addr, &sha);
|
||||
let (runner, at_once) = net.runner(RunOn::Cpu, 2);
|
||||
assert_eq!(at_once, 2 + 4, "four server slots on top of two local threads");
|
||||
assert!(worst(&want, &runner.run(&input).unwrap()) < 5e-3);
|
||||
let d = net.device(RunOn::Cpu);
|
||||
assert_eq!(d["remote"]["state"], "used", "{d}");
|
||||
assert_eq!(d["remote"]["device"], "CPU (test server)");
|
||||
// a local card faster than the server there and back (here a stand-in that takes no time): nothing is shared,
|
||||
// the photo never waits for the slower server, and the status says why
|
||||
let mut net = with_remote(&path, 64, &addr, &sha);
|
||||
net.gpu = OnceLock::from(Ok(GpuSide { runner: Box::new(cpu.clone()), adapter: "Fast card".into(), card_ms: 1e-6, cpu_ms: 1000.0 }));
|
||||
let (_, at_once) = net.runner(RunOn::Gpu, 8);
|
||||
assert_eq!(at_once, GPU_PARALLEL, "no server slots");
|
||||
let d = net.device(RunOn::Gpu);
|
||||
assert_eq!(d["remote"]["state"], "slower", "{d}");
|
||||
assert!(d["remote"]["remoteMs"].as_f64().is_some_and(|ms| ms > 0.0) && d["remote"]["localMs"].as_f64().is_some(), "{d}");
|
||||
// a server running another model is refused: everything runs here
|
||||
let other = server(lightcraft_denoise::runtime::CpuRunner::load(&path, &cpu.manifest().clone()).unwrap(), "0000", 64);
|
||||
let net = with_remote(&path, 64, &other, &sha);
|
||||
let (runner, at_once) = net.runner(RunOn::Cpu, 2);
|
||||
assert_eq!(at_once, 2);
|
||||
assert_eq!(runner.run(&input).unwrap(), want, "the CPU path, bit for bit");
|
||||
let d = net.device(RunOn::Cpu);
|
||||
assert!(d["remote"]["reason"].as_str().is_some_and(|r| r.contains("another model")), "{d}");
|
||||
// nobody listening: the same
|
||||
let net = with_remote(&path, 64, "127.0.0.1:1", &sha);
|
||||
assert_eq!(net.runner(RunOn::Cpu, 3).1, 3);
|
||||
assert_eq!(net.device(RunOn::Cpu)["remote"]["state"], "unavailable");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_the_processor_chosen_everything_runs_on_the_cpu() {
|
||||
let (model, cpu, path) = model("cpu", 64);
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let (runner, at_once) = model.runner(RunOn::Cpu, 16);
|
||||
assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit");
|
||||
assert_eq!(at_once, 16, "the threads asked for are the threads used");
|
||||
assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu");
|
||||
assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_card_is_used_where_it_is_faster_than_the_processor() {
|
||||
// measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms
|
||||
for threads in [1, 2, 8, 16, 64] {
|
||||
assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}");
|
||||
}
|
||||
assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets");
|
||||
assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen");
|
||||
assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster");
|
||||
assert!(!card_is_faster(10.0, 0.0, 1));
|
||||
}
|
||||
|
||||
/// A card that gives numbers that are not finite.
|
||||
struct Garbage(AtomicUsize);
|
||||
|
||||
impl TileRunner for Garbage {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
let n = input.len() / 4 * 12;
|
||||
Ok(vec![f32::NAN; n])
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() {
|
||||
let (_, cpu, path) = model("garbage", 64);
|
||||
let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 };
|
||||
let model = Network {
|
||||
cpu: cpu.clone(),
|
||||
path: path.clone(),
|
||||
tile: 32,
|
||||
gpu: OnceLock::from(Ok(card)),
|
||||
fallbacks: AtomicUsize::new(0),
|
||||
last_threads: AtomicUsize::new(0),
|
||||
remote: None,
|
||||
local_card: AtomicBool::new(true),
|
||||
};
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
let (runner, at_once) = model.runner(RunOn::Auto, 8);
|
||||
assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen");
|
||||
for _ in 0..GPU_MAX_FALLBACKS {
|
||||
assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile");
|
||||
}
|
||||
let device = model.device(RunOn::Auto);
|
||||
assert_eq!(device["kind"], "cpu", "{device}");
|
||||
assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}");
|
||||
assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS);
|
||||
let (_, at_once) = model.runner(RunOn::Gpu, 8);
|
||||
assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slow_card_is_left_out_unless_it_is_chosen() {
|
||||
let (_, cpu, path) = model("slow", 64);
|
||||
let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 };
|
||||
let model = Network {
|
||||
cpu,
|
||||
path: path.clone(),
|
||||
tile: 32,
|
||||
gpu: OnceLock::from(Ok(slow)),
|
||||
fallbacks: AtomicUsize::new(0),
|
||||
last_threads: AtomicUsize::new(0),
|
||||
remote: None,
|
||||
local_card: AtomicBool::new(true),
|
||||
};
|
||||
assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it");
|
||||
let device = model.device(RunOn::Auto);
|
||||
assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer")));
|
||||
assert_eq!(device["cardMs"], 490.0, "the times measured are reported");
|
||||
assert_eq!(device["retry"], false, "trying again would not make it faster");
|
||||
assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not");
|
||||
assert_eq!(model.device(RunOn::Auto)["kind"], "gpu");
|
||||
assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_marker_left_behind_keeps_the_card_off() {
|
||||
let (model, _, path) = model("crashed", 64);
|
||||
let marker = path.with_file_name(GPU_SETUP_MARKER);
|
||||
std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap();
|
||||
let (_, at_once) = model.runner(RunOn::Gpu, 4);
|
||||
assert_eq!(at_once, 4, "the CPU does the work");
|
||||
let device = model.device(RunOn::Gpu);
|
||||
assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}");
|
||||
assert_eq!(device["retry"], true, "the interface offers to try again");
|
||||
assert!(marker.exists(), "it stays until the user chooses where denoise runs");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() {
|
||||
let dir = folder("setup");
|
||||
let marker = dir.join(GPU_SETUP_MARKER);
|
||||
let seen = Arc::new(AtomicBool::new(false));
|
||||
let (m, s) = (marker.clone(), seen.clone());
|
||||
let r = set_up_card(
|
||||
&marker,
|
||||
Duration::from_secs(10),
|
||||
move || {
|
||||
s.store(m.exists(), Ordering::Relaxed);
|
||||
Ok(7)
|
||||
},
|
||||
|| "cpu",
|
||||
);
|
||||
assert_eq!(r, Ok((7, "cpu")));
|
||||
assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up");
|
||||
assert!(!marker.exists(), "and gone after");
|
||||
// a set-up that fails or panics removes it too
|
||||
assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into()));
|
||||
assert!(!marker.exists());
|
||||
let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ());
|
||||
assert!(r.is_err() && !marker.exists(), "{r:?}");
|
||||
// a marker found before the set-up starts means the last one took LightCraft down: the card is not touched
|
||||
std::fs::write(&marker, "left behind").unwrap();
|
||||
let touched = Arc::new(AtomicBool::new(false));
|
||||
let t = touched.clone();
|
||||
let r = set_up_card(
|
||||
&marker,
|
||||
Duration::from_secs(10),
|
||||
move || {
|
||||
t.store(true, Ordering::Relaxed);
|
||||
Ok(())
|
||||
},
|
||||
|| (),
|
||||
);
|
||||
assert_eq!(r, Err(GPU_CRASHED.to_string()));
|
||||
assert!(!touched.load(Ordering::Relaxed) && marker.exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_that_takes_too_long_is_given_up_on() {
|
||||
let dir = folder("slowsetup");
|
||||
let marker = dir.join(GPU_SETUP_MARKER);
|
||||
let started = web_time::Instant::now();
|
||||
let slow = || -> Result<(), String> {
|
||||
std::thread::sleep(Duration::from_millis(600));
|
||||
Ok(())
|
||||
};
|
||||
let r = set_up_card(&marker, Duration::from_millis(50), slow, || ());
|
||||
assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}");
|
||||
assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card");
|
||||
// while the first is still stuck, another set-up of the same model does not start a second one
|
||||
let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ());
|
||||
assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e)));
|
||||
// once it ends its marker goes, and the card can be set up again
|
||||
while marker.exists() {
|
||||
assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended");
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) {
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2)));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
mod gpu_tests;
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
use super::*;
|
||||
use lightcraft_denoise::manifest::{Domain, Gain};
|
||||
|
||||
/// A folder of the test's own: the card's set-up leaves its marker beside the model.
|
||||
fn folder(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
/// The model file is read again when the card is set up (for the first photo), so it stays until the test ends.
|
||||
fn model(name: &str, tile: u64) -> (Arc<dyn Model>, lightcraft_denoise::runtime::CpuRunner, PathBuf) {
|
||||
let path = folder(name).join("model.onnx");
|
||||
std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap();
|
||||
let manifest = DenoiserManifest {
|
||||
id: "synthetic".into(),
|
||||
name: "Synthetic".into(),
|
||||
version: "1".into(),
|
||||
licence: Default::default(),
|
||||
source: None,
|
||||
sha256: None,
|
||||
size_bytes: None,
|
||||
provenance: String::new(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: tile as u32,
|
||||
overlap: 16,
|
||||
gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },
|
||||
};
|
||||
let loaded = default_loader()(&path, &manifest).unwrap();
|
||||
let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap();
|
||||
(loaded, cpu, path)
|
||||
}
|
||||
|
||||
fn worst(want: &[f32], got: &[f32]) -> f32 {
|
||||
assert_eq!(want.len(), got.len());
|
||||
let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs()));
|
||||
want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tile_on_the_card_is_the_cpus_tile() {
|
||||
let (model, cpu, path) = model("same", 64);
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
let (runner, at_once) = model.runner(RunOn::Gpu, 16);
|
||||
assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3);
|
||||
let device = model.device(RunOn::Gpu);
|
||||
eprintln!("device: {device}");
|
||||
match device["kind"].as_str() {
|
||||
Some("gpu") => {
|
||||
assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty()));
|
||||
assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once");
|
||||
assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}");
|
||||
}
|
||||
Some("cpu") => {
|
||||
assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}");
|
||||
assert_eq!(at_once, 16);
|
||||
}
|
||||
other => panic!("the first photo decides where the work runs, not {other:?}"),
|
||||
}
|
||||
assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
/// A server on this computer running `cpu` for the model with `sha256`.
|
||||
fn server(cpu: lightcraft_denoise::runtime::CpuRunner, sha256: &str, tile: usize) -> String {
|
||||
use lightcraft_denoise::remote::{ModelId, ServerOptions, serve};
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = l.local_addr().unwrap().to_string();
|
||||
let id = ModelId { id: "synthetic".into(), version: "1".into(), sha256: sha256.into() };
|
||||
let run: lightcraft_denoise::remote::RunTile = Arc::new(move |input: &[f32]| cpu.run(input));
|
||||
let opts = ServerOptions { token: "remote-test-token-0123".into(), ..Default::default() };
|
||||
std::thread::spawn(move || serve(l, id, tile, "CPU (test server)".into(), run, opts, Arc::new(|_| {})));
|
||||
addr
|
||||
}
|
||||
|
||||
fn with_remote(path: &Path, tile: u64, addr: &str, sha256: &str) -> Network {
|
||||
let manifest = DenoiserManifest {
|
||||
id: "synthetic".into(),
|
||||
name: "Synthetic".into(),
|
||||
version: "1".into(),
|
||||
licence: Default::default(),
|
||||
source: None,
|
||||
sha256: None,
|
||||
size_bytes: None,
|
||||
provenance: String::new(),
|
||||
domain: Domain::BayerToRgb,
|
||||
tile: tile as u32,
|
||||
overlap: 16,
|
||||
gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 },
|
||||
};
|
||||
let remote = RemoteSide::new(lightcraft_denoise::remote::Remote::new(addr, "remote-test-token-0123", sha256, tile as usize));
|
||||
network(path, &manifest, Some(remote)).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_remote_server_shares_the_tiles_and_drops_out_when_it_cannot() {
|
||||
let (_, cpu, path) = model("remote", 64);
|
||||
let sha = lightcraft_denoise::hash::sha256_file(&path).unwrap();
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
// the same model on the server: its slots come on top of this computer's threads, answers within f16 rounding
|
||||
let addr = server(lightcraft_denoise::runtime::CpuRunner::load(&path, &cpu.manifest().clone()).unwrap(), &sha, 64);
|
||||
let net = with_remote(&path, 64, &addr, &sha);
|
||||
let (runner, at_once) = net.runner(RunOn::Cpu, 2);
|
||||
assert_eq!(at_once, 2 + 4, "four server slots on top of two local threads");
|
||||
assert!(worst(&want, &runner.run(&input).unwrap()) < 5e-3);
|
||||
let d = net.device(RunOn::Cpu);
|
||||
assert_eq!(d["remote"]["state"], "used", "{d}");
|
||||
assert_eq!(d["remote"]["device"], "CPU (test server)");
|
||||
// a local card faster than the server there and back (here a stand-in that takes no time): nothing is shared,
|
||||
// the photo never waits for the slower server, and the status says why
|
||||
let mut net = with_remote(&path, 64, &addr, &sha);
|
||||
net.gpu = OnceLock::from(Ok(GpuSide { runner: Box::new(cpu.clone()), adapter: "Fast card".into(), card_ms: 1e-6, cpu_ms: 1000.0 }));
|
||||
let (_, at_once) = net.runner(RunOn::Gpu, 8);
|
||||
assert_eq!(at_once, GPU_PARALLEL, "no server slots");
|
||||
let d = net.device(RunOn::Gpu);
|
||||
assert_eq!(d["remote"]["state"], "slower", "{d}");
|
||||
assert!(d["remote"]["remoteMs"].as_f64().is_some_and(|ms| ms > 0.0) && d["remote"]["localMs"].as_f64().is_some(), "{d}");
|
||||
// a server running another model is refused: everything runs here
|
||||
let other = server(lightcraft_denoise::runtime::CpuRunner::load(&path, &cpu.manifest().clone()).unwrap(), "0000", 64);
|
||||
let net = with_remote(&path, 64, &other, &sha);
|
||||
let (runner, at_once) = net.runner(RunOn::Cpu, 2);
|
||||
assert_eq!(at_once, 2);
|
||||
assert_eq!(runner.run(&input).unwrap(), want, "the CPU path, bit for bit");
|
||||
let d = net.device(RunOn::Cpu);
|
||||
assert!(d["remote"]["reason"].as_str().is_some_and(|r| r.contains("another model")), "{d}");
|
||||
// nobody listening: the same
|
||||
let net = with_remote(&path, 64, "127.0.0.1:1", &sha);
|
||||
assert_eq!(net.runner(RunOn::Cpu, 3).1, 3);
|
||||
assert_eq!(net.device(RunOn::Cpu)["remote"]["state"], "unavailable");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_the_processor_chosen_everything_runs_on_the_cpu() {
|
||||
let (model, cpu, path) = model("cpu", 64);
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let (runner, at_once) = model.runner(RunOn::Cpu, 16);
|
||||
assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit");
|
||||
assert_eq!(at_once, 16, "the threads asked for are the threads used");
|
||||
assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu");
|
||||
assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_card_is_used_where_it_is_faster_than_the_processor() {
|
||||
// measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms
|
||||
for threads in [1, 2, 8, 16, 64] {
|
||||
assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}");
|
||||
}
|
||||
assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets");
|
||||
assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen");
|
||||
assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster");
|
||||
assert!(!card_is_faster(10.0, 0.0, 1));
|
||||
}
|
||||
|
||||
/// A card that gives numbers that are not finite.
|
||||
struct Garbage(AtomicUsize);
|
||||
|
||||
impl TileRunner for Garbage {
|
||||
fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
let n = input.len() / 4 * 12;
|
||||
Ok(vec![f32::NAN; n])
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() {
|
||||
let (_, cpu, path) = model("garbage", 64);
|
||||
let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 };
|
||||
let model = Network {
|
||||
cpu: cpu.clone(),
|
||||
path: path.clone(),
|
||||
tile: 32,
|
||||
gpu: OnceLock::from(Ok(card)),
|
||||
fallbacks: AtomicUsize::new(0),
|
||||
last_threads: AtomicUsize::new(0),
|
||||
remote: None,
|
||||
local_card: AtomicBool::new(true),
|
||||
};
|
||||
let (input, _) = lightcraft_denoise::runtime::test_tile(64);
|
||||
let want = cpu.run(&input).unwrap();
|
||||
let (runner, at_once) = model.runner(RunOn::Auto, 8);
|
||||
assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen");
|
||||
for _ in 0..GPU_MAX_FALLBACKS {
|
||||
assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile");
|
||||
}
|
||||
let device = model.device(RunOn::Auto);
|
||||
assert_eq!(device["kind"], "cpu", "{device}");
|
||||
assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}");
|
||||
assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS);
|
||||
let (_, at_once) = model.runner(RunOn::Gpu, 8);
|
||||
assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slow_card_is_left_out_unless_it_is_chosen() {
|
||||
let (_, cpu, path) = model("slow", 64);
|
||||
let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 };
|
||||
let model = Network {
|
||||
cpu,
|
||||
path: path.clone(),
|
||||
tile: 32,
|
||||
gpu: OnceLock::from(Ok(slow)),
|
||||
fallbacks: AtomicUsize::new(0),
|
||||
last_threads: AtomicUsize::new(0),
|
||||
remote: None,
|
||||
local_card: AtomicBool::new(true),
|
||||
};
|
||||
assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it");
|
||||
let device = model.device(RunOn::Auto);
|
||||
assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer")));
|
||||
assert_eq!(device["cardMs"], 490.0, "the times measured are reported");
|
||||
assert_eq!(device["retry"], false, "trying again would not make it faster");
|
||||
assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not");
|
||||
assert_eq!(model.device(RunOn::Auto)["kind"], "gpu");
|
||||
assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_marker_left_behind_keeps_the_card_off() {
|
||||
let (model, _, path) = model("crashed", 64);
|
||||
let marker = path.with_file_name(GPU_SETUP_MARKER);
|
||||
std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap();
|
||||
let (_, at_once) = model.runner(RunOn::Gpu, 4);
|
||||
assert_eq!(at_once, 4, "the CPU does the work");
|
||||
let device = model.device(RunOn::Gpu);
|
||||
assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}");
|
||||
assert_eq!(device["retry"], true, "the interface offers to try again");
|
||||
assert!(marker.exists(), "it stays until the user chooses where denoise runs");
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() {
|
||||
let dir = folder("setup");
|
||||
let marker = dir.join(GPU_SETUP_MARKER);
|
||||
let seen = Arc::new(AtomicBool::new(false));
|
||||
let (m, s) = (marker.clone(), seen.clone());
|
||||
let r = set_up_card(
|
||||
&marker,
|
||||
Duration::from_secs(10),
|
||||
move || {
|
||||
s.store(m.exists(), Ordering::Relaxed);
|
||||
Ok(7)
|
||||
},
|
||||
|| "cpu",
|
||||
);
|
||||
assert_eq!(r, Ok((7, "cpu")));
|
||||
assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up");
|
||||
assert!(!marker.exists(), "and gone after");
|
||||
// a set-up that fails or panics removes it too
|
||||
assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into()));
|
||||
assert!(!marker.exists());
|
||||
let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ());
|
||||
assert!(r.is_err() && !marker.exists(), "{r:?}");
|
||||
// a marker found before the set-up starts means the last one took LightCraft down: the card is not touched
|
||||
std::fs::write(&marker, "left behind").unwrap();
|
||||
let touched = Arc::new(AtomicBool::new(false));
|
||||
let t = touched.clone();
|
||||
let r = set_up_card(
|
||||
&marker,
|
||||
Duration::from_secs(10),
|
||||
move || {
|
||||
t.store(true, Ordering::Relaxed);
|
||||
Ok(())
|
||||
},
|
||||
|| (),
|
||||
);
|
||||
assert_eq!(r, Err(GPU_CRASHED.to_string()));
|
||||
assert!(!touched.load(Ordering::Relaxed) && marker.exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_set_up_that_takes_too_long_is_given_up_on() {
|
||||
let dir = folder("slowsetup");
|
||||
let marker = dir.join(GPU_SETUP_MARKER);
|
||||
let started = web_time::Instant::now();
|
||||
let slow = || -> Result<(), String> {
|
||||
std::thread::sleep(Duration::from_millis(600));
|
||||
Ok(())
|
||||
};
|
||||
let r = set_up_card(&marker, Duration::from_millis(50), slow, || ());
|
||||
assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}");
|
||||
assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card");
|
||||
// while the first is still stuck, another set-up of the same model does not start a second one
|
||||
let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ());
|
||||
assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e)));
|
||||
// once it ends its marker goes, and the card can be set up again
|
||||
while marker.exists() {
|
||||
assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended");
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) {
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2)));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
Reference in New Issue
Block a user