LR-EDIT-DETAIL-DENOISE: address installation and model-policy review

Generated with Codex.
This commit is contained in:
Daniel Voshart
2026-10-08 22:07:23 -04:00
parent 7935999d94
commit fd4019300a
59 changed files with 1437 additions and 217 deletions
+151 -35
View File
@@ -7,7 +7,7 @@
//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
use std::path::{Path, PathBuf};
use std::sync::PoisonError;
use std::sync::{Arc, PoisonError, mpsc};
use lightcraft_catalog::PhotoId;
use lightcraft_denoise::archive;
@@ -18,7 +18,7 @@ use serde_json::{Value, json};
use super::{CommandSpec, always, bad, bool_or, cmd, str_param};
use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};
use crate::model_download::{self, State};
use crate::model_download::State;
use crate::{EngineError, Result, Session};
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
@@ -129,8 +129,16 @@ fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result<Source> {
/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing
/// a model is how someone says they want it. `verified` is for a download whose hash was already checked.
fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activate: bool, verified: bool) -> Result<Value> {
let dir = models_dir(s, c)?;
fn install_file(
dir: &Path,
loader: &crate::denoise::Loader,
run_on: RunOn,
accepted_at: String,
c: &str,
src: &Path,
acknowledged: bool,
verified: bool,
) -> Result<(Value, Arc<dyn crate::denoise::Model>)> {
let ident = identify(src, c, None)?;
let m = ident.manifest.clone();
manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
@@ -141,7 +149,11 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa
));
}
let home = dir.join(&m.id);
if home.join("model.onnx").exists() {
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
let mut cleanup = InstallCleanup { home: home.clone(), armed: true };
let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
let _ = std::fs::remove_file(&part);
let placed: Result<()> = match ident.entry {
@@ -162,26 +174,24 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa
return Err(e);
}
};
let _ = std::fs::remove_file(&final_path);
if let Err(e) = std::fs::rename(&part, &final_path) {
let _ = std::fs::remove_file(&part);
return Err(fail("could not finish the copy", e));
}
// a model that does not work is never left installed
let run_on = s.denoise.settings.run_on();
let test = (s.denoise.loader)(&final_path, &m).and_then(|model| model.self_test(run_on));
let test = match test {
// Test the staged file first. A failed replacement preserves the installed model.
let model = loader(&part, &m).map_err(|e| bad(c, e))?;
let test = match model.self_test(run_on) {
Ok(t) => t,
Err(e) => {
let _ = std::fs::remove_file(&final_path);
let _ = std::fs::remove_file(home.join("denoise-model.json"));
let _ = std::fs::remove_dir(&home);
let _ = std::fs::remove_file(&part);
return Err(bad(c, e));
}
};
// An installed id is immutable: installing again must never destroy a usable model.
if final_path.exists() {
let _ = std::fs::remove_file(&part);
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::rename(&part, &final_path).map_err(|e| fail("could not finish the copy", e))?;
write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?;
let accepted = json!({
"acceptedAt": (s.clock)(),
"acceptedAt": accepted_at,
"licence": m.licence.name,
"commercial": m.licence.commercial,
"fileName": ident.file_name,
@@ -189,26 +199,116 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa
"selfTest": test,
});
write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?;
// a file that was downloaded and waited for the user's acceptance has served: the installed copy is the model now
if src.parent() == Some(dir.join(model_download::STAGING).as_path()) {
let _ = std::fs::remove_file(src);
s.denoise.downloads.staged_file_used(src);
cleanup.armed = false;
Ok((json!({"installed": row(&m, known::find(&m.id).and_then(|k| host_of(&k)), true, false, &accepted), "model": m.id}), model))
}
/// Removes only files created for this new installation, including on a worker panic.
struct InstallCleanup {
home: PathBuf,
armed: bool,
}
impl Drop for InstallCleanup {
fn drop(&mut self) {
if self.armed {
for name in ["model.onnx.part", "model.onnx", "denoise-model.json", "installed.json"] {
let _ = std::fs::remove_file(self.home.join(name));
}
let _ = std::fs::remove_dir(&self.home);
}
}
}
pub(crate) struct InstallJob {
id: String,
activate: bool,
result: mpsc::Receiver<std::result::Result<(Value, Arc<dyn crate::denoise::Model>), String>>,
}
fn activate_install(s: &mut Session, result: &Value, model: Arc<dyn crate::denoise::Model>, activate: bool) -> Result<Value> {
let dir = models_dir(s, "denoise.models.install")?;
let id = result["installed"]["id"].as_str().ok_or_else(|| bad("denoise.models.install", "missing installed id"))?;
let mut st = read_settings(&dir);
if activate {
st.model = Some(m.id.clone());
st.model = Some(id.to_string());
write_settings(&dir, &st).map_err(EngineError::Other)?;
}
s.denoise.touch();
s.denoise_refresh_active(true);
let host = known::find(&m.id).and_then(|k| host_of(&k));
Ok(json!({"installed": row(&m, host, true, st.model.as_deref() == Some(m.id.as_str()), &accepted), "model": st.model}))
if let Some(a) = &s.denoise.active
&& a.id == id
{
*a.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(model);
}
let mut v = result.clone();
v["installed"]["selected"] = json!(st.model.as_deref() == Some(id));
v["model"] = json!(st.model);
Ok(v)
}
fn start_install(s: &mut Session, src: PathBuf, id: String, activate: bool, verified: bool) -> Result<Value> {
const C: &str = "denoise.models.install";
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let dir = models_dir(s, C)?;
let loader = s.denoise.loader.clone();
let run_on = s.denoise.settings.run_on();
let at = (s.clock)();
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("denoise-model-install".into())
.stack_size(16 * 1024 * 1024)
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _held = crate::memory::work_gate().acquire(512 * 1024 * 1024);
install_file(&dir, &loader, run_on, at, C, &src, true, verified).map_err(|e| plain(&e))
}))
.unwrap_or_else(|_| Err("the model installation stopped unexpectedly".into()));
if verified {
let _ = std::fs::remove_file(&src);
}
let _ = tx.send(outcome);
})
.map_err(|e| bad(C, format!("could not start installation: {e}")))?;
s.denoise.install = Some(InstallJob { id: id.clone(), activate, result: rx });
s.denoise.downloads.set_outcome(&id, State::Installing);
Ok(json!({"started": id}))
}
fn install(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.install";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
install_file(s, C, Path::new(path), bool_or(p, "acknowledged", false), bool_or(p, "activate", true), false)
let acknowledged = bool_or(p, "acknowledged", false);
let activate = bool_or(p, "activate", true);
if !acknowledged {
return Err(bad(C, "the licence has not been accepted: show its terms first"));
}
if bool_or(p, "background", false) {
return start_install(s, PathBuf::from(path), "local-install".into(), activate, false);
}
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let (result, model) =
install_file(&models_dir(s, C)?, &s.denoise.loader, s.denoise.settings.run_on(), (s.clock)(), C, Path::new(path), acknowledged, false)?;
activate_install(s, &result, model, activate)
}
/// Read only the small adjacent manifest; hashing and execution happen in the install worker.
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.inspect";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let src = Path::new(path);
if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) {
return Err(bad(C, "choose an .onnx file with denoise-model.json beside it"));
}
let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES)
.ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?;
let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?;
let dir = models_dir(s, C)?;
Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m,
"sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()}))
}
/// What an error says, without the command it came from (for a line shown to the user).
@@ -222,17 +322,31 @@ fn plain(e: &EngineError) -> String {
/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without
/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`.
pub(crate) fn finish_downloads(s: &mut Session) {
const C: &str = "denoise.models.download";
for (id, path, _sha) in s.denoise.downloads.finished() {
match install_file(s, C, &path, true, true, true) {
Ok(_) => s.denoise.downloads.set_outcome(&id, State::Installed),
Err(e) => {
// a file that cannot be installed is of no use to anyone: throw it away, say why
let _ = std::fs::remove_file(&path);
s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));
}
if let Some(job) = &s.denoise.install {
let outcome = match job.result.try_recv() {
Ok(v) => Some(v),
Err(mpsc::TryRecvError::Empty) => None,
Err(mpsc::TryRecvError::Disconnected) => Some(Err("the installation worker stopped".into())),
};
if let Some(outcome) = outcome
&& let Some(job) = s.denoise.install.take()
{
let state = match outcome {
Ok((v, model)) => match activate_install(s, &v, model, job.activate) {
Ok(_) => State::Installed,
Err(e) => State::Failed(plain(&e)),
},
Err(e) => State::Failed(e),
};
s.denoise.downloads.set_outcome(&job.id, state);
}
}
if s.denoise.install.is_none()
&& let Some((id, path, _sha)) = s.denoise.downloads.finished().into_iter().next()
&& let Err(e) = start_install(s, path, id.clone(), true, true)
{
s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));
}
}
fn download(s: &mut Session, p: &Value) -> Result<Value> {
@@ -263,6 +377,7 @@ fn download_row(id: &str, state: &State) -> Value {
match state {
State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
State::Installing => json!({"id": id, "state": "installing", "from": from}),
State::Installed => json!({"id": id, "state": "installed", "from": from}),
State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
@@ -517,7 +632,8 @@ fn clear(s: &mut Session, _: &Value) -> Result<Value> {
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list),
cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, model} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),
cmd!(query "denoise.models.inspect", "Inspect Denoise Model", [], None, "{path} → model terms from an adjacent manifest; never executes the file", always, inspect),
cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true, background?: false} → {installed, model} or {started} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),
cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download),
cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads),
cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
+22 -7
View File
@@ -98,6 +98,10 @@ pub(crate) trait Model: Send + Sync {
/// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics
/// card is set up and checked too, and the result says where the work will run.
fn self_test(&self, run_on: RunOn) -> Result<Value, String>;
/// Conservative scratch bound for an unknown runner.
fn bytes_per_tile(&self) -> usize {
512 * 1024 * 1024
}
/// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not
/// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were
/// timed. Never starts anything.
@@ -215,7 +219,7 @@ fn set_up_card<T: Send + 'static, R>(
running.push(marker.to_path_buf());
SetUpEnded(marker.to_path_buf())
};
let started = std::time::Instant::now();
let started = web_time::Instant::now();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::Builder::new()
.name("denoise-gpu-setup".into())
@@ -241,10 +245,10 @@ fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 {
if runner.run(input).is_err() {
return f64::INFINITY;
}
let began = std::time::Instant::now();
let began = web_time::Instant::now();
let mut best = f64::INFINITY;
for _ in 0..3 {
let one = std::time::Instant::now();
let one = web_time::Instant::now();
if runner.run(input).is_err() {
break;
}
@@ -268,7 +272,7 @@ fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usi
Ok((runner, got, card_ms))
};
let on_cpu = || {
let started = std::time::Instant::now();
let started = web_time::Instant::now();
(cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0)
};
let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?;
@@ -351,6 +355,9 @@ impl TileRunner for Network {
#[cfg(feature = "denoise")]
impl Model for Network {
fn bytes_per_tile(&self) -> usize {
self.cpu.bytes_per_tile()
}
fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
let threads = threads.max(1);
self.last_threads.store(threads, Ordering::Relaxed);
@@ -689,7 +696,7 @@ pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Resul
if product::is_current(&spec.product, &spec.key) {
return Ok(false);
}
let began = std::time::Instant::now();
let began = web_time::Instant::now();
let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed));
if cancelled() {
return Err(MakeError::Cancelled);
@@ -726,7 +733,13 @@ pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Resul
}
let decoded = began.elapsed();
let model = loaded(spec)?;
let (runner, parallel) = model.runner(spec.run_on, spec.parallel.max(1));
// Packed input, returned RGB and the preceding group's RGB overlap while tiles run.
let scratch =
model.bytes_per_tile().saturating_add(128usize.saturating_mul(spec.manifest.tile as usize).saturating_mul(spec.manifest.tile as usize));
let (_, budget) = crate::memory::work_gate().usage();
let parallel = spec.parallel.clamp(1, 4).min((budget / scratch.max(1)).max(1));
let _held = crate::memory::work_gate().acquire(mosaic.data.len().saturating_mul(20).saturating_add(scratch.saturating_mul(parallel)));
let (runner, parallel) = model.runner(spec.run_on, parallel);
let params = Params {
tile: spec.manifest.tile as usize,
overlap: spec.manifest.overlap as usize,
@@ -859,6 +872,7 @@ pub(crate) struct State {
pub models_dir: Option<PathBuf>,
/// Models being downloaded at the user's request.
pub downloads: Downloads,
pub(crate) install: Option<crate::cmd::denoise::InstallJob>,
/// How a model is loaded (tests put another in).
pub loader: Loader,
pub(crate) active: Option<Active>,
@@ -883,6 +897,7 @@ impl Default for State {
State {
models_dir: None,
downloads: Downloads::default(),
install: None,
loader: default_loader(),
active: None,
active_seen: None,
@@ -1604,7 +1619,7 @@ mod gpu_tests {
fn a_set_up_that_takes_too_long_is_given_up_on() {
let dir = folder("slowsetup");
let marker = dir.join(GPU_SETUP_MARKER);
let started = std::time::Instant::now();
let started = web_time::Instant::now();
let slow = || -> Result<(), String> {
std::thread::sleep(Duration::from_millis(600));
Ok(())
+21 -12
View File
@@ -309,8 +309,11 @@ fn load_bytes_now(
let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));
// camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied,
// fitted to `max_edge` and upright (the same for the plain and the denoised picture)
let finish = |mut img: Rgb32f| -> Rgb32f {
let finish = |mut img: Rgb32f| {
let mut stages = vec![("develop", t0.elapsed())];
stages.push(("transform", t0.elapsed()));
lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP);
stages.push(("highlights", t0.elapsed()));
img.map_in_place(|p| {
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
let rgb = [
@@ -324,21 +327,27 @@ fn load_bytes_now(
};
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
});
stages.push(("colour", t0.elapsed()));
let img = fit(&img, max_edge, max_edge, Filter::Box);
img.into_oriented(raw.orientation)
stages.push(("fit", t0.elapsed()));
let img = img.into_oriented(raw.orientation);
stages.push(("orient", t0.elapsed()));
(img, stages)
};
let develop_ms = t0.elapsed();
let img = finish(img);
let twin = twin.map(finish);
let (img, stages) = finish(img);
if lightcraft_pipeline::profiling() {
let finish_ms = (t0.elapsed() - develop_ms).as_secs_f64() * 1e3;
eprintln!(
"[profile] raw source {}×{} (max {max_edge}): develop {:.1} ms, finish {finish_ms:.1} ms (after decode)",
img.width,
img.height,
develop_ms.as_secs_f64() * 1e3
);
let mut prev = std::time::Duration::ZERO;
let parts: Vec<String> = stages
.iter()
.map(|(n, t)| {
let d = *t - prev;
prev = *t;
format!("{n} {:.1}", d.as_secs_f64() * 1e3)
})
.collect();
eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
}
let twin = twin.map(|picture| finish(picture).0);
let (temp, tint) = xy_to_temp_tint(xy);
let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve));
+4
View File
@@ -10,6 +10,10 @@
//! run them off the UI thread.
#![forbid(unsafe_code)]
// Model/cache types remain available without linking the optional inference crate.
#[cfg(not(feature = "denoise"))]
extern crate lightcraft_denoise_core as lightcraft_denoise;
pub mod availability;
mod camera_preview;
pub mod camera_profiles;
+17 -4
View File
@@ -156,10 +156,9 @@ impl Twin {
Twin { image, mix: std::sync::Mutex::new(None) }
}
/// Bytes held, the last mix included.
/// Bytes reserved for the twin and its lazy mix, before either cache cost can grow.
fn bytes(&self) -> usize {
let mixed = self.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |m| m.1.data.len() * 12);
self.image.data.len() * 12 + mixed
self.image.data.len().saturating_mul(24)
}
}
@@ -207,7 +206,7 @@ impl DecodedSource {
mixed
}
/// Bytes held: the picture, its denoised twin and the last mix.
/// Cache cost: the picture, its denoised twin and room for the lazy mix.
pub fn bytes(&self) -> usize {
self.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + self.denoised.as_ref().map_or(0, |t| t.bytes())
}
@@ -1390,6 +1389,20 @@ mod thumbnail_hash_tests {
mod tests {
use super::*;
#[test]
fn denoise_mix_is_reserved_before_the_source_enters_the_cache() {
let mut plain = Rgb32f::new(8, 8);
plain.data.fill([1.0; 3]);
let mut source = DecodedSource::new(Arc::new(plain), None);
source.denoised = Some(Arc::new(Twin::new(Arc::new(Rgb32f::new(8, 8)))));
let reserved = source.bytes();
assert!(reserved >= 3 * 8 * 8 * 12);
assert_eq!(source.image_for(0.5).data[0], [0.5; 3]);
assert_eq!(source.bytes(), reserved, "creating a mix cannot evade the LRU's recorded cost");
let _ = source.image_for(0.25);
assert_eq!(source.bytes(), reserved, "changing Amount replaces the reserved mix");
}
#[test]
fn decoder_info_survives_render_jobs_cache_and_eviction() {
let tone = lightcraft_pipeline::tone::CameraTone::new(std::array::from_fn(|i| {
+4 -8
View File
@@ -35,6 +35,7 @@ pub enum State {
sha256: String,
},
/// Installed and in use: nothing more to do but tell the user.
Installing,
Installed,
Failed(String),
Cancelled,
@@ -107,7 +108,7 @@ impl Downloads {
job.cancel.store(true, Ordering::Relaxed);
match job.get() {
// the thread notices, deletes the partial file and marks itself cancelled
State::Running { .. } => {}
State::Running { .. } | State::Installing => {}
State::Done { path, .. } => {
let _ = std::fs::remove_file(path);
self.jobs.remove(id);
@@ -119,11 +120,6 @@ impl Downloads {
true
}
/// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted.
pub fn staged_file_used(&mut self, path: &Path) {
self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path));
}
/// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).
pub fn finished(&self) -> Vec<(String, PathBuf, String)> {
let mut out: Vec<_> = self
@@ -157,7 +153,7 @@ impl Drop for Downloads {
for job in self.jobs.values() {
job.cancel.store(true, Ordering::Relaxed);
}
let started = std::time::Instant::now();
let started = web_time::Instant::now();
while self.running() && started.elapsed() < Duration::from_secs(1) {
std::thread::sleep(Duration::from_millis(10));
}
@@ -336,7 +332,7 @@ mod tests {
let mut d = Downloads::default();
d.jobs.insert("m".into(), shared.clone());
assert!(d.running());
let started = std::time::Instant::now();
let started = web_time::Instant::now();
drop(d);
assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");
assert_eq!(shared.get(), State::Cancelled);
+61 -1
View File
@@ -81,6 +81,65 @@ fn model_files(dir: &Path, id: &str) -> PathBuf {
src.join("model.onnx")
}
#[test]
fn model_installation_keeps_the_frame_pump_responsive_and_reuses_the_loaded_model() {
use std::sync::{Mutex, mpsc};
use std::time::Duration;
let mut x = setup("background-install", false);
let path = model_files(&x.dir, "stand-in");
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let release_rx = Mutex::new(release_rx);
let runs = x.runs.clone();
let loads = Arc::new(AtomicUsize::new(0));
let worker_loads = loads.clone();
x.s.denoise.loader = Arc::new(move |_, _| {
worker_loads.fetch_add(1, Ordering::Relaxed);
entered_tx.send(()).map_err(|e| e.to_string())?;
release_rx.lock().unwrap().recv_timeout(Duration::from_secs(3)).map_err(|e| e.to_string())?;
Ok(Arc::new(Dim { runs: runs.clone() }))
});
let info = x.s.execute("denoise.models.inspect", &json!({"path": path})).unwrap();
assert_eq!(info["model"]["id"], "stand-in");
assert_eq!(loads.load(Ordering::Relaxed), 0, "showing terms never executes the model");
let r = x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).unwrap();
assert_eq!(r["started"], "local-install");
entered_rx.recv_timeout(Duration::from_secs(3)).unwrap();
// The loader is blocked: the frame pump must still return, reporting installation in progress.
let state = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
assert_eq!(state["downloads"][0]["state"], "installing");
assert_eq!(x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap()["active"], false);
assert!(x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).is_err());
release_tx.send(()).unwrap();
for _ in 0..600 {
let v = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
if v["downloads"][0]["state"] == "installed" {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(x.s.execute("denoise.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed");
let a = x.s.denoise.active.as_ref().unwrap();
assert!(a.model.lock().unwrap().is_some(), "the worker's tested model is reused");
assert_eq!(loads.load(Ordering::Relaxed), 1);
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
#[test]
fn failed_model_replacement_preserves_the_installed_model() {
let mut x = setup("replacement", true);
let installed = x.dir.join("denoise-models/stand-in/model.onnx");
let original = std::fs::read(&installed).unwrap();
let source = model_files(&x.dir, "stand-in");
std::fs::write(&source, "a different broken model").unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": source, "acknowledged": true})).is_err());
assert_eq!(std::fs::read(installed).unwrap(), original);
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["model"], "stand-in");
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one.
fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec<u8> {
let data: Vec<u16> = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect();
@@ -339,7 +398,8 @@ fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() {
let mut x = setup("fails", true);
let id = photo(&x.s);
x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into()));
// (the model chosen is read afresh: it was loaded by the install's self-test only)
// Simulate loss of the in-memory model; successful installation now retains its tested instance.
*x.s.denoise.active.as_ref().unwrap().model.lock().unwrap() = None;
set_amount(&mut x.s, 80.0);
let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100}));
let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();