Merge the faces stack (#172/#173/#174, integrate-batch-14) into AI RAW denoise (#384)

One config-dir helper (engine::config::config_dir; camera_profiles re-exports it) with both
the face-models and denoise-models defaults; hosts set up both model folders; Settings has
Faces and AI Denoise tabs; both command sets registered; both model dialogs; locale entries
from both; one [features] table per app manifest; Cargo.lock regenerated by cargo; the
DecodedSource cache cost counts both segmentation mattes and the denoised twin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-09 00:29:59 -04:00
co-authored by Claude Opus 5.5
112 changed files with 15313 additions and 236 deletions
+184
View File
@@ -0,0 +1,184 @@
//! `faces.detect`: find faces with the YuNet detector the user has downloaded (Settings > Faces).
//!
//! The photo is rendered upright and uncropped with default settings (so edits, crops and rotations you have
//! made do not matter), the detector runs on it, and the faces come back as fractions of the photo's width
//! and height, the same frame face regions from XMP use. Unless `apply` is false, they are added to the photo as
//! unnamed face regions in one undoable step; earlier detections are replaced, and regions that came from XMP (or
//! that you drew or named) are never touched. Like every region edit it is catalog-only: no sidecar is written.
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Instant;
use lightcraft_catalog::Op;
use lightcraft_develop::DevelopSettings;
use lightcraft_faces::known::YUNET_ID;
use lightcraft_faces::yunet::{Detector, Face, Options};
use lightcraft_meta::{Region, RegionKind};
use serde_json::{Value, json};
use super::{CommandSpec, bad, cmd, has_selection};
use crate::{Result, Session};
const C: &str = "faces.detect";
/// Regions made by this command say so in their description; only those are replaced on a new run.
pub(crate) const MARK: &str = "Detected by ";
pub(crate) const LABEL: &str = "YuNet 2023mar";
/// Long edge of the picture the detector looks at (it shrinks it to its own 640 anyway).
const LOOK_EDGE: usize = 1280;
/// YuNet is 232 KB; a file this much larger in its folder is not it.
const MAX_MODEL_BYTES: u64 = 8 << 20;
/// How much a detection must overlap a region the photo already has to count as the same face.
const EXISTING_IOU: f64 = 0.3;
/// What to tell someone who has not downloaded the detector yet.
const NOT_INSTALLED: &str = "finding faces needs the YuNet face detector (232 KB, MIT): download it in Settings > Faces";
/// The installed YuNet, loaded once and kept (a new one is loaded when the model folder changes).
/// (also used by the background scan, which aligns faces by the detector's landmarks and finds faces in photos that have none)
pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, String> {
static LOADED: Mutex<Option<(PathBuf, Arc<Detector>)>> = Mutex::new(None);
let home = s.face_models_dir.as_deref().ok_or("this build has nowhere to keep face models (the desktop app does)")?.join(YUNET_ID);
let path = home.join("model.onnx");
// installing writes the manifest last, so a folder without one is not (yet) a model
if !path.is_file() || !home.join("face-model.json").is_file() {
return Err(NOT_INSTALLED.into());
}
let mut loaded = LOADED.lock().unwrap_or_else(PoisonError::into_inner);
if let Some((p, d)) = loaded.as_ref()
&& *p == path
{
return Ok(d.clone());
}
let len = std::fs::metadata(&path).map_err(|e| format!("could not read the detector: {e}"))?.len();
if len > MAX_MODEL_BYTES {
return Err("the detector's file is far bigger than YuNet: remove it in Settings > Faces and download it again".into());
}
let bytes = std::fs::read(&path).map_err(|e| format!("could not read the detector: {e}"))?;
let d = Arc::new(Detector::new(&bytes).map_err(|e| format!("{e}: remove the detector in Settings > Faces and download it again"))?);
*loaded = Some((path, d.clone()));
Ok(d)
}
fn face_json(f: &Face) -> Value {
json!({
"rect": {"x0": f.x0, "y0": f.y0, "x1": f.x1, "y1": f.y1},
"score": f.score,
"landmarks": f.landmarks.iter().map(|(x, y)| json!([x, y])).collect::<Vec<_>>(),
})
}
/// A detected face that lies on a region the photo already has (from XMP, drawn or named) is that region, not a new face.
fn overlaps_existing(f: &Face, regions: &[Region]) -> bool {
regions.iter().any(|r| {
let (w, h) = (
(r.rect.x1.min(f64::from(f.x1)) - r.rect.x0.max(f64::from(f.x0))).max(0.0),
(r.rect.y1.min(f64::from(f.y1)) - r.rect.y0.max(f64::from(f.y0))).max(0.0),
);
let inter = w * h;
let union = (r.rect.x1 - r.rect.x0) * (r.rect.y1 - r.rect.y0) + f64::from((f.x1 - f.x0) * (f.y1 - f.y0)) - inter;
union > 0.0 && inter / union >= EXISTING_IOU
})
}
pub(crate) fn is_detected(r: &Region) -> bool {
r.description.as_deref().is_some_and(|d| d.starts_with(MARK))
}
/// The photo as the faces see it: upright, uncropped, default settings, as 8-bit RGB with its size, its long edge at
/// most `edge`.
pub(crate) fn render_rgb(s: &mut Session, id: lightcraft_catalog::PhotoId, edge: usize) -> std::result::Result<(Vec<u8>, usize, usize), String> {
let settings = DevelopSettings::default();
let job = s.preview_job(id, edge, edge, false, &settings).ok_or("no such photo")?;
let image = job.run().rendered?.image;
let rgb: Vec<u8> = image.data.iter().flat_map(|px| [px[0], px[1], px[2]]).collect();
Ok((rgb, image.width, image.height))
}
fn detect(s: &mut Session, p: &Value) -> Result<Value> {
let apply = p.get("apply").and_then(Value::as_bool).unwrap_or(true);
let defaults = Options::default();
let opts = Options {
score: p.get("score").and_then(Value::as_f64).map_or(defaults.score, |v| v as f32),
nms_iou: p.get("nmsIou").and_then(Value::as_f64).map_or(defaults.nms_iou, |v| v as f32),
max_faces: p.get("maxFaces").and_then(Value::as_u64).map_or(defaults.max_faces, |v| usize::try_from(v).unwrap_or(defaults.max_faces)),
};
let detector = detector(s).map_err(|e| bad(C, e))?;
let started = Instant::now();
let (mut results, mut ops) = (Vec::new(), Vec::new());
for id in s.targets(p) {
let (rgb, width, height) = match render_rgb(s, id, LOOK_EDGE) {
Ok(r) => r,
Err(e) => {
results.push(json!({"id": id.0, "error": e}));
continue;
}
};
let faces = match detector.detect(&rgb, width, height, &opts) {
Ok(f) => f,
Err(e) => {
results.push(json!({"id": id.0, "error": e.to_string()}));
continue;
}
};
if apply && let Some(photo) = s.catalog.photo(id) {
let mut meta = photo.meta.clone();
meta.regions.retain(|r| !is_detected(r));
let fresh: Vec<&Face> = faces.iter().filter(|f| !overlaps_existing(f, &meta.regions)).collect();
for f in fresh {
meta.regions.push(Region {
rect: lightcraft_geom::Rect { x0: f64::from(f.x0), y0: f64::from(f.y0), x1: f64::from(f.x1), y1: f64::from(f.y1) },
kind: RegionKind::Face,
name: None,
description: Some(format!("{MARK}{LABEL}")),
});
}
ops.push(Op::SetMeta { id, meta: Box::new(meta) });
}
results.push(json!({"id": id.0, "faces": faces.iter().map(face_json).collect::<Vec<_>>()}));
}
let applied = !ops.is_empty();
if applied {
s.commit("Detect Faces", Op::Batch { ops })?;
// regions are not written to XMP, so a sidecar must not be rewritten for this
s.skip_auto_write = true;
}
Ok(json!({"detector": LABEL, "applied": applied, "photos": results, "ms": started.elapsed().as_secs_f64() * 1000.0}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![cmd!(
"faces.detect",
"Detect Faces",
["Photo"],
None,
"{ids?, apply?: bool (default true), score?: 0.6, nmsIou?: 0.3, maxFaces?} → {photos: [{id, faces: [{rect, score, landmarks}]}]}. Finds faces with the YuNet detector (download it first: `faces.models.download {id: \"yunet-2023mar\"}`) and, unless `apply` is false, adds them as unnamed face regions (one undo step), replacing earlier detections but never regions from XMP or ones you named; the sidecar is not touched",
has_selection,
detect
)]
}
#[cfg(test)]
mod tests {
use super::*;
fn face(x0: f32, y0: f32, x1: f32, y1: f32) -> Face {
Face { x0, y0, x1, y1, score: 0.9, landmarks: [(0.0, 0.0); 5] }
}
fn region(x0: f64, y0: f64, x1: f64, y1: f64) -> Region {
Region { rect: lightcraft_geom::Rect { x0, y0, x1, y1 }, kind: RegionKind::Face, name: Some("Ann".into()), description: None }
}
#[test]
fn a_detection_on_an_existing_region_is_not_a_new_face() {
let existing = [region(0.40, 0.20, 0.60, 0.50), region(0.10, 0.10, 0.20, 0.20)];
// the same face, a bit larger or shifted (a Lightroom box is not YuNet's box)
assert!(overlaps_existing(&face(0.38, 0.18, 0.64, 0.54), &existing));
assert!(overlaps_existing(&face(0.42, 0.22, 0.58, 0.48), &existing));
// another face, a sliver of overlap, and no regions at all
assert!(!overlaps_existing(&face(0.70, 0.20, 0.90, 0.50), &existing));
assert!(!overlaps_existing(&face(0.59, 0.49, 0.80, 0.80), &existing));
assert!(!overlaps_existing(&face(0.0, 0.0, 0.0, 0.0), &existing), "an empty box overlaps nothing");
assert!(!overlaps_existing(&face(0.4, 0.2, 0.6, 0.5), &[]));
}
}
+511
View File
@@ -0,0 +1,511 @@
//! Face model commands (`faces.*`): which models exist, inspecting a file the user dropped in, installing
//! it once its licence is accepted, removing it, and the faces on/off setting.
//!
//! Models live in the folder the host provides (`Session::face_models_dir`): one subfolder per model with
//! `model.onnx`, `face-model.json` (its manifest) and `installed.json` (what the user accepted), plus
//! `settings.json` beside them. Model files and manifests are hostile input: sizes are capped, ids are
//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
use std::io::Read;
use std::path::{Path, PathBuf};
use lightcraft_faces::catalog::{self, Catalog};
use lightcraft_faces::hash::sha256_file;
use lightcraft_faces::manifest::{self, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES, ModelManifest, Role};
use lightcraft_faces::suggest::{Suggestion, suggest_with};
use lightcraft_faces::{known, onnx};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use super::{CommandSpec, always, bad, cmd, str_param};
use crate::face_download::{self, State};
use crate::{EngineError, Result, Session};
#[derive(Default, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub(super) struct FaceSettings {
pub(super) enabled: bool,
/// The installed recogniser in use.
pub(super) embedder: Option<String>,
}
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
EngineError::Other(format!("{what}: {e}"))
}
fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {
s.face_models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep face models (the desktop app does)"))
}
/// Read at most `max` bytes of a small file.
fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {
let mut buf = Vec::new();
std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;
(buf.len() <= max).then_some(buf)
}
pub(super) fn read_settings(dir: &Path) -> FaceSettings {
read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
}
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
let part = path.with_extension("part");
std::fs::write(&part, bytes).map_err(|e| fail(&format!("could not write {}", path.display()), e))?;
std::fs::rename(&part, path).map_err(|e| {
let _ = std::fs::remove_file(&part);
fail(&format!("could not write {}", path.display()), e)
})
}
fn write_settings(dir: &Path, st: &FaceSettings) -> Result<()> {
std::fs::create_dir_all(dir).map_err(|e| fail("could not create the face models folder", e))?;
write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| fail("settings", e))?)
}
pub(super) struct Installed {
pub(super) manifest: ModelManifest,
pub(super) accepted: Value,
}
/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that
/// does not validate or does not match its folder) is ignored, never trusted.
pub(super) fn installed_models(dir: &Path) -> Vec<Installed> {
let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
let mut out = Vec::new();
for e in entries.flatten() {
let p = e.path();
let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };
if !manifest::valid_id(&name) || !p.is_dir() || !p.join("model.onnx").is_file() {
continue;
}
let Some(m) = read_capped(&p.join("face-model.json"), MAX_MANIFEST_BYTES).and_then(|b| manifest::parse(&b).ok()) else { continue };
if m.id != name {
continue;
}
let accepted = read_capped(&p.join("installed.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
out.push(Installed { manifest: m, accepted });
}
out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));
out
}
/// Read the user's catalog (`catalog.json` in the models folder) anew: a few kilobytes, so it is always current.
fn refresh_catalog(s: &mut Session) {
let Some(dir) = s.face_models_dir.clone() else { return };
let file = dir.join("catalog.json");
s.face_catalog = match read_capped(&file, catalog::MAX_CATALOG_BYTES) {
Some(bytes) => catalog::parse(&bytes),
None if file.is_file() => catalog::parse(&vec![b' '; catalog::MAX_CATALOG_BYTES + 1]),
None => Catalog::default(),
};
}
/// The manifests of the catalog's models (what a file's hash is matched against, besides the built-in ones).
fn catalog_manifests(s: &Session) -> Vec<ModelManifest> {
s.face_catalog.entries.iter().map(|e| e.manifest.clone()).collect()
}
/// How to fetch a model: the built-in address, or the one in the user's catalog.
fn download_spec(s: &Session, id: &str) -> Option<known::Download> {
known::download(id).or_else(|| s.face_catalog.entries.iter().find(|e| e.manifest.id == id).and_then(catalog::Entry::download))
}
fn row(s: &Session, m: &ModelManifest, installed: bool, selected: bool, accepted: &Value) -> Value {
let from_catalog = s.face_catalog.entries.iter().any(|e| e.manifest.id == m.id);
json!({
"id": m.id,
"name": m.name,
"role": m.role,
"licence": m.licence,
"provenance": m.provenance,
"source": m.source,
"sizeBytes": m.size_bytes,
"sha256": m.sha256,
// how fast it is as a multiple of a ResNet-100 model (computers differ, ratios between models hardly do), and
// that in words; null when nobody has measured it
"speed": m.speed,
"speedText": m.speed.map(manifest::describe_speed).filter(|t| !t.is_empty()),
"known": from_catalog || known::all().iter().any(|k| k.id == m.id),
// from the user's own catalog file
"fromCatalog": from_catalog,
// a pinned address LightCraft can fetch it from (the user presses Download), and which site that is
"downloadHost": download_spec(s, &m.id).map(|d| known::host(&d.url).to_string()),
"installed": installed,
"selected": selected,
"accepted": accepted,
})
}
struct Inspection {
sha256: String,
size: u64,
file_name: String,
suggestion: Suggestion,
}
/// Look at a model file. `known_sha` is its SHA-256 when something already checked it (a finished download), so a
/// file of hundreds of megabytes is not read again just to be hashed.
fn inspect_file(path: &str, cmd: &str, known_sha: Option<&str>, extra: &[ModelManifest]) -> Result<Inspection> {
let p = Path::new(path);
let meta = std::fs::metadata(p).map_err(|e| bad(cmd, format!("cannot read `{path}`: {e}")))?;
if !meta.is_file() {
return Err(bad(cmd, format!("`{path}` is not a file")));
}
if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {
return Err(bad(cmd, "a model file must be between 1 byte and 512 MiB"));
}
let sha256 = match known_sha {
Some(h) => h.to_string(),
None => sha256_file(p).map_err(|e| fail(&format!("could not read {path}"), e))?,
};
let file_name = p.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let suggestion = match onnx::probe_path(p) {
Ok(info) => suggest_with(&info, &sha256, meta.len(), &file_name, extra),
Err(e) => Suggestion::Unsupported(e.to_string()),
};
Ok(Inspection { sha256, size: meta.len(), file_name, suggestion })
}
fn list(s: &mut Session, _: &Value) -> Result<Value> {
refresh_catalog(s);
let dir = s.face_models_dir.clone();
let (settings, installed) = match &dir {
Some(d) => (read_settings(d), installed_models(d)),
None => (FaceSettings::default(), Vec::new()),
};
let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);
let mut models: Vec<Value> = Vec::new();
for k in known::all() {
let found = on_disk(&k.id);
models.push(row(s, &k, found.is_some(), settings.embedder.as_deref() == Some(k.id.as_str()), found.map_or(&Value::Null, |f| &f.accepted)));
}
// the user's own catalog
for e in &s.face_catalog.entries {
let found = on_disk(&e.manifest.id);
let chosen = settings.embedder.as_deref() == Some(e.manifest.id.as_str());
models.push(row(s, &e.manifest, found.is_some(), chosen, found.map_or(&Value::Null, |f| &f.accepted)));
}
for i in &installed {
let listed = known::all().iter().any(|k| k.id == i.manifest.id) || s.face_catalog.entries.iter().any(|e| e.manifest.id == i.manifest.id);
if !listed {
models.push(row(s, &i.manifest, true, settings.embedder.as_deref() == Some(i.manifest.id.as_str()), &i.accepted));
}
}
Ok(json!({
"dir": dir.as_ref().map(|d| d.display().to_string()),
"enabled": settings.enabled,
"embedder": settings.embedder,
// Native builds always have the pure-Rust runner; model files/background scanning remain native-only.
"runtime": cfg!(not(target_arch = "wasm32")),
"models": models,
// the user's own list of models to download: where it goes, how many it holds, and what was wrong with the rest
"catalog": {
"path": dir.as_ref().map(|d| d.join("catalog.json").display().to_string()),
"models": s.face_catalog.entries.len(),
"errors": s.face_catalog.errors,
},
}))
}
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.inspect";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
refresh_catalog(s);
let ins = inspect_file(path, C, None, &catalog_manifests(s))?;
let installed = s.face_models_dir.as_deref().map(installed_models).unwrap_or_default();
let (kind, model, assumptions, reason) = match &ins.suggestion {
Suggestion::Known(m) => ("known", Some(m), Vec::new(), None),
Suggestion::Draft { manifest, assumptions } => ("draft", Some(manifest), assumptions.clone(), None),
Suggestion::Unsupported(why) => ("unsupported", None, Vec::new(), Some(why.clone())),
};
let already = model.is_some_and(|m| installed.iter().any(|i| i.manifest.sha256 == m.sha256 && i.manifest.id == m.id));
Ok(json!({
"path": path,
"fileName": ins.file_name,
"sizeBytes": ins.size,
"sha256": ins.sha256,
"kind": kind,
"model": model.map(|m| row(s, m, already, false, &Value::Null)),
"assumptions": assumptions,
"reason": reason,
"alreadyInstalled": already,
}))
}
fn install(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.install";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let acknowledged = p.get("acknowledged").and_then(Value::as_bool).unwrap_or(false);
let activate = p.get("activate").and_then(Value::as_bool).unwrap_or(true);
refresh_catalog(s);
let ins = inspect_file(path, C, None, &catalog_manifests(s))?;
install_file(s, C, Path::new(path), ins, acknowledged, activate, false)
}
/// Install the inspected model file `src`. A recognition model that passes its self-test becomes the one in use and
/// recognition is switched on (`activate`, the default): installing a model is how someone says they want it.
/// `verified` is for a download that was already checked against its recorded hash: it is moved into place instead
/// of being copied and read again.
fn install_file(s: &mut Session, c: &str, src: &Path, ins: Inspection, acknowledged: bool, activate: bool, verified: bool) -> Result<Value> {
let dir = models_dir(s, c)?;
let m = match ins.suggestion {
Suggestion::Known(m) | Suggestion::Draft { manifest: m, .. } => m,
Suggestion::Unsupported(why) => return Err(bad(c, format!("this model cannot be used yet: {why}"))),
};
if !matches!(m.role, Role::Embedder | Role::Detector) {
return Err(bad(c, "only face detection and recognition models can be installed"));
}
manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
if !acknowledged {
return Err(bad(c, "the licence has not been accepted: show the user its terms and pass `acknowledged: true` once they agree"));
}
let home = dir.join(&m.id);
std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
let moved = verified && std::fs::rename(src, &part).is_ok();
let placed = if moved {
Ok(())
} else {
std::fs::copy(src, &part).map_err(|e| fail("could not copy the model (is the disk full?)", e)).and_then(|_| match sha256_file(&part) {
Ok(h) if h == ins.sha256 => Ok(()),
Ok(_) => Err(EngineError::Other("the copy does not match the original (the file changed while copying?)".into())),
Err(e) => Err(fail("could not check the copy", e)),
})
};
if let Err(e) = placed {
let _ = std::fs::remove_file(&part);
return Err(e);
}
// Validate the candidate before replacing a working installation.
let test = match self_test(&part, &m) {
Ok(t) => t,
Err(e) => {
let _ = std::fs::remove_file(&part);
let _ = std::fs::remove_dir(&home);
return Err(bad(c, e));
}
};
if let Err(e) = std::fs::rename(&part, &final_path) {
let _ = std::fs::remove_file(&part);
return Err(fail("could not finish the copy", e));
}
write_atomic(&home.join("face-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?)?;
let accepted = json!({"acceptedAt": (s.clock)(), "licence": m.licence.name, "commercial": m.licence.commercial, "fileName": ins.file_name, "selfTest": test});
write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?)?;
// a file that was downloaded and waited in the staging folder has served: the installed copy is the model now
if src.parent() == Some(dir.join(face_download::STAGING).as_path()) {
let _ = std::fs::remove_file(src);
s.face_downloads.staged_file_used(src);
}
// the newest model is the one in use (the earlier one stays installed, and `faces.models.select` switches back)
if activate && m.role == Role::Embedder {
let mut st = read_settings(&dir);
st.embedder = Some(m.id.clone());
st.enabled = true;
write_settings(&dir, &st)?;
}
let now = read_settings(&dir);
let selected = now.embedder.as_deref() == Some(m.id.as_str());
Ok(json!({"installed": row(s, &m, true, selected, &accepted), "active": {"embedder": now.embedder, "enabled": now.enabled}}))
}
/// What an error says, without the command it came from (for a line shown to the user).
fn plain(e: &EngineError) -> String {
match e {
EngineError::BadParams { msg, .. } => msg.clone(),
other => other.to_string(),
}
}
/// Install every download that has arrived: each was accepted when it was started, so it is installed, chosen and
/// switched on without another question. Called by `faces.models.downloads` and by every frame's `faces.pump`.
pub(super) fn finish_downloads(s: &mut Session) {
const C: &str = "faces.models.download";
let extra = catalog_manifests(s);
for (id, path, sha) in s.face_downloads.finished() {
let outcome = inspect_file(&path.to_string_lossy(), C, Some(&sha), &extra).and_then(|ins| install_file(s, C, &path, ins, true, true, true));
match outcome {
Ok(_) => s.face_downloads.set_outcome(&id, State::Installed),
Err(e) => {
// a file that cannot be installed is of no use to anyone: throw it away, say why
let _ = std::fs::remove_file(&path);
s.face_downloads.set_outcome(&id, State::Failed(plain(&e)));
}
}
}
}
/// `faces.models.download {id, acknowledged: true}`: fetch a model LightCraft knows an address for, in the background;
/// when it has arrived and checked out it is installed.
fn download(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.download";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
refresh_catalog(s);
let spec = download_spec(s, id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
if cfg!(target_arch = "wasm32") {
return Err(bad(C, "the web build downloads no models"));
}
if p.get("acknowledged").and_then(Value::as_bool) != Some(true) {
return Err(bad(
C,
"the licence has not been accepted: show the user the model's terms (see `faces.models.list`) and pass `acknowledged: true` once they agree",
));
}
if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {
return Err(bad(C, "that model is already installed"));
}
let host = spec.host().to_string();
s.face_downloads.start(spec, &dir).map_err(|e| bad(C, e))?;
Ok(json!({"started": id, "from": host}))
}
fn download_row(s: &Session, id: &str, state: &State) -> Value {
let from = download_spec(s, id).map(|d| d.host().to_string());
match state {
State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
State::Installed => json!({"id": id, "state": "installed", "from": from}),
State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
}
}
/// `faces.models.downloads`: where each download stands (a finished one is installed first).
fn downloads(s: &mut Session, _: &Value) -> Result<Value> {
finish_downloads(s);
let all: Vec<Value> = s.face_downloads.snapshot().iter().map(|(id, st)| download_row(s, id, st)).collect();
Ok(json!({"running": s.face_downloads.running(), "downloads": all}))
}
/// `faces.models.downloadCancel {id}`: stop a download, or clear a finished one (a staged file is deleted).
fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.downloadCancel";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
Ok(json!({"discarded": s.face_downloads.discard(id)}))
}
/// Load the model and run its self-test: `Ok(null)` in a build without the recognition runtime (nothing can
/// be checked), `Ok(result)` when it passes, `Err(reason)` when it does not load or fails a check.
#[cfg(not(target_arch = "wasm32"))]
fn self_test(path: &Path, m: &ModelManifest) -> std::result::Result<Value, String> {
if m.role == Role::Detector {
let bytes = read_capped(path, 8 << 20).ok_or("the detector cannot be read within its 8 MiB limit")?;
let detector = lightcraft_faces::yunet::Detector::new(&bytes).map_err(|e| e.to_string())?;
if Some(detector.input_side()) != usize::try_from(m.input.width).ok()
|| m.input.width != m.input.height
|| !matches!(&m.output, manifest::OutputSpec::Detector { decoder } if decoder == "yunet-v2")
{
return Err("unsupported detector input or output decoder".into());
}
detector.detect(&[0u8; 12], 2, 2, &lightcraft_faces::yunet::Options::default()).map_err(|e| e.to_string())?;
return Ok(json!({"ok": true}));
}
let embedder = lightcraft_faces::runtime::Embedder::load(path, m).map_err(|e| e.to_string())?;
let t = embedder.self_test();
if !t.ok {
let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();
return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));
}
serde_json::to_value(&t).map_err(|e| e.to_string())
}
#[cfg(target_arch = "wasm32")]
fn self_test(_: &Path, _: &ModelManifest) -> std::result::Result<Value, String> {
Ok(Value::Null)
}
/// `faces.models.test {id}`: run an installed model's self-test again.
fn test(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.test";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !cfg!(not(target_arch = "wasm32")) {
return Err(bad(C, "this build cannot run face recognition models"));
}
let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;
if found.manifest.role != Role::Embedder {
return Err(bad(C, "only recognition models have a self-test"));
}
let result = self_test(&dir.join(id).join("model.onnx"), &found.manifest);
let (ok, detail) = match &result {
Ok(v) => (true, v.clone()),
Err(e) => (false, json!(e)),
};
// keep the latest outcome with the model's record
let mut record = found.accepted;
if let Some(o) = record.as_object_mut() {
o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });
write_atomic(&dir.join(id).join("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)?;
}
Ok(json!({"id": id, "ok": ok, "result": detail}))
}
fn remove(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.remove";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !manifest::valid_id(id) {
return Err(bad(C, "not a model id"));
}
let home = dir.join(id);
// only a folder that is a model (it has a manifest) is ever deleted
if !home.join("face-model.json").is_file() {
return Err(bad(C, "that model is not installed"));
}
std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;
let mut st = read_settings(&dir);
if st.embedder.as_deref() == Some(id) {
// another installed recogniser takes over, so recognition keeps working; with none left nothing is chosen
st.embedder = installed_models(&dir).into_iter().find(|i| i.manifest.id != id && i.manifest.role == Role::Embedder).map(|i| i.manifest.id);
write_settings(&dir, &st)?;
}
Ok(json!({"removed": id, "embedder": st.embedder}))
}
fn select(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.models.select";
let dir = models_dir(s, C)?;
let id = match p.get("id") {
None | Some(Value::Null) => None,
Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),
};
if let Some(id) = &id
&& !installed_models(&dir).iter().any(|i| &i.manifest.id == id && i.manifest.role == Role::Embedder)
{
return Err(bad(C, "that recognition model is not installed"));
}
let mut st = read_settings(&dir);
st.embedder = id;
write_settings(&dir, &st)?;
Ok(json!({"embedder": st.embedder}))
}
fn enable(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "faces.enable";
let dir = models_dir(s, C)?;
let mut st = read_settings(&dir);
st.enabled = p.get("enabled").and_then(Value::as_bool).unwrap_or(!st.enabled);
write_settings(&dir, &st)?;
// the background scan notices at once, not within the second it trusts its last look
#[cfg(not(target_arch = "wasm32"))]
{
s.faces.enabled_seen = None;
}
Ok(json!({"enabled": st.enabled}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(query "faces.models.list", "Face Models", [], None, "{} → {dir, enabled, embedder, runtime, models: [{id, name, role, licence{name, commercial, url, notice}, provenance, source, sizeBytes, known, installed, selected}]}", always, list),
cmd!(query "faces.models.inspect", "Inspect Face Model File", [], None, "{path} → what a .onnx file is: {kind: known | draft | unsupported, model, assumptions, reason, alreadyInstalled}; installs nothing", always, inspect),
cmd!(query "faces.models.install", "Install Face Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, active: {embedder, enabled}} — copy a .onnx face recognition model into the models folder. `acknowledged` must be true: the user has been shown its licence (see inspect) and accepted it. Unless `activate` is false the model becomes the one in use and recognition is switched on; an earlier model stays installed", always, install),
cmd!(query "faces.models.download", "Download Face Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a recognition model LightCraft has a pinned address for (see `downloadHost` in the list), in the background over pure-Rust https (lightcraft-fetch). `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, then installed, chosen and switched on by itself; `faces.models.downloads` shows how far it is", always, download),
cmd!(query "faces.models.downloads", "Face Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived; `installed` stays listed until `faces.models.downloadCancel` clears it", always, downloads),
cmd!(query "faces.models.downloadCancel", "Cancel Face Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
cmd!(query "faces.models.test", "Test Face Model", [], None, "{id} → {ok, result: {loadMs, embedMs, dimension, checks}} — load an installed recognition model and check it gives sensible faces; needs the recognition runtime", always, test),
cmd!(query "faces.models.remove", "Remove Face Model", [], None, "{id} — delete an installed model", always, remove),
cmd!(query "faces.models.select", "Choose Face Recognition Model", [], None, "{id: installed recogniser | null}", always, select),
cmd!(query "faces.enable", "Face Recognition On/Off", [], None, "{enabled?: bool} (toggles when omitted)", always, enable),
]
}
File diff suppressed because it is too large Load Diff
+6
View File
@@ -14,6 +14,9 @@ pub(crate) mod denoise;
mod develop;
mod edit;
mod export;
pub(crate) mod face_detect;
mod face_models;
mod face_recognize;
pub mod filters;
pub mod keywords;
pub mod library;
@@ -142,6 +145,9 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(metadata::specs());
v.extend(filters::specs());
v.extend(denoise::specs());
v.extend(face_models::specs());
v.extend(face_detect::specs());
v.extend(face_recognize::specs());
v
})
}