From 6fee4fbacd8689a6e93f47132ba2b36969137a2d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 3 Nov 2026 17:46:15 +0000 Subject: [PATCH 1/3] security(lensdb): bound the names calibrations copy, read database files through the opened regular file --- crates/lensdb/src/lib.rs | 30 +++++++++++++++++++++++++----- crates/lensdb/src/tests.rs | 25 ++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/lensdb/src/lib.rs b/crates/lensdb/src/lib.rs index cf6b2ec145..e2910125c0 100644 --- a/crates/lensdb/src/lib.rs +++ b/crates/lensdb/src/lib.rs @@ -30,6 +30,9 @@ pub use model::{Distortion, Tca, Vignetting}; const MAX_FILE_BYTES: u64 = 16 << 20; /// Most files read from one directory. const MAX_FILES: usize = 2000; +/// Most bytes of lens names that profiles may copy per parsed document (each calibration owns a copy): a real +/// database copies about a megabyte. +const MAX_PROFILE_COPY_BYTES: usize = 32 << 20; /// Largest radial error (half-diagonal units) accepted when converting a distortion to the DNG polynomial. pub const MAX_WARP_ERROR: f64 = 0.001; /// Largest relative error accepted when converting vignetting to the DNG polynomial. @@ -144,7 +147,8 @@ impl LensDb { let root = xml::parse(xml)?; let Some(top) = root.kid("lensdatabase") else { return Err(Error::Xml("no element".into())) }; let cameras = top.kids("camera").filter_map(camera).collect(); - let lenses = top.kids("lens").flat_map(lens).collect(); + let mut budget = MAX_PROFILE_COPY_BYTES; + let lenses = top.kids("lens").flat_map(|n| lens(n, &mut budget)).collect(); Ok(LensDb { cameras, lenses }) } @@ -272,11 +276,22 @@ fn exif_focal_range(exif: &[String]) -> Option<(f64, f64)> { } fn read_capped(path: &Path) -> Result { - let len = std::fs::metadata(path).map_err(|e| Error::Io(e.to_string()))?.len(); - if len > MAX_FILE_BYTES { + use std::io::Read; + let io = |e: std::io::Error| Error::Io(e.to_string()); + // the opened file is checked, not the pathname: a link or FIFO swapped in after a pathname check is refused too + let file = std::fs::File::open(path).map_err(io)?; + let meta = file.metadata().map_err(io)?; + if !meta.is_file() { + return Err(Error::Io(format!("{}: not a regular file", path.display()))); + } + if meta.len() > MAX_FILE_BYTES { + return Err(Error::TooLarge); + } + let mut bytes = Vec::new(); + file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes).map_err(io)?; + if bytes.len() as u64 > MAX_FILE_BYTES { return Err(Error::TooLarge); } - let bytes = std::fs::read(path).map_err(|e| Error::Io(e.to_string()))?; Ok(String::from_utf8_lossy(&bytes).into_owned()) } @@ -369,7 +384,7 @@ fn positive(v: Option) -> Option { } /// One `` element → one profile per `` (newer files give each its own crop factor and aspect). -fn lens(n: &xml::Node) -> Vec { +fn lens(n: &xml::Node, copy_budget: &mut usize) -> Vec { let Some(maker) = n.kid_text("maker") else { return Vec::new() }; let models: Vec = n.kids("model").map(|m| m.text.trim().to_string()).filter(|m| !m.is_empty()).collect(); if models.is_empty() { @@ -379,7 +394,12 @@ fn lens(n: &xml::Node) -> Vec { let aspect = aspect_of(n.kid_text("aspect-ratio")).unwrap_or(1.5); let rectilinear = n.kid_text("type").is_none_or(|t| t.eq_ignore_ascii_case("rectilinear")); let mounts = n.kids("mount").map(|m| m.text.trim().to_string()).filter(|m| !m.is_empty()).collect::>(); + // every calibration owns a copy of the lens's names: calibrations past the document's shared copy budget are left out + let meta_bytes = maker.len() + models.iter().chain(&mounts).map(|m| m.len() + 1).sum::() + 1; + let take = n.kids("calibration").count().min(*copy_budget / meta_bytes); + *copy_budget -= take * meta_bytes; n.kids("calibration") + .take(take) .map(|cal| { let mut p = Profile { maker: maker.to_string(), diff --git a/crates/lensdb/src/tests.rs b/crates/lensdb/src/tests.rs index 71c9a4a0eb..033111de7f 100644 --- a/crates/lensdb/src/tests.rs +++ b/crates/lensdb/src/tests.rs @@ -232,7 +232,8 @@ fn fisheyes_get_no_rectilinear_warp_and_bad_shots_none() { #[test] fn load_dir_reads_xml_files_and_reports_bad_ones() { let dir = std::env::temp_dir().join(format!("lensdb-test-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); + // exclusive: an existing (possibly planted) entry fails the test instead of being written through + std::fs::create_dir(&dir).unwrap(); std::fs::write(dir.join("a.xml"), DB).unwrap(); std::fs::write(dir.join("b.xml"), ""; + let xml = format!("M{name}{}", cal.repeat(2000)); + let db = LensDb::parse(&xml).unwrap(); + assert!(!db.lenses.is_empty()); + assert!(db.lenses.len() * 400_000 <= MAX_PROFILE_COPY_BYTES); +} + +#[cfg(unix)] +#[test] +fn load_dir_refuses_a_non_regular_xml_entry() { + let dir = std::env::temp_dir().join(format!("lensdb-test-dev-{}", std::process::id())); + std::fs::create_dir(&dir).unwrap(); + std::os::unix::fs::symlink("/dev/zero", dir.join("z.xml")).unwrap(); + let (db, errs) = LensDb::load_dir(&dir).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert!(db.is_empty()); + assert_eq!(errs.len(), 1, "{errs:?}"); +} From f7e43a4a45dd4381fb70366797b1df76d0f7f444 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 3 Nov 2026 23:58:25 +0000 Subject: [PATCH 2/3] address review: charge name allocations and only kept calibrations to the copy budget --- crates/lensdb/src/lib.rs | 107 +++++++++++++++++++++---------------- crates/lensdb/src/tests.rs | 23 ++++++++ 2 files changed, 83 insertions(+), 47 deletions(-) diff --git a/crates/lensdb/src/lib.rs b/crates/lensdb/src/lib.rs index e2910125c0..91f6723c1c 100644 --- a/crates/lensdb/src/lib.rs +++ b/crates/lensdb/src/lib.rs @@ -33,6 +33,8 @@ const MAX_FILES: usize = 2000; /// Most bytes of lens names that profiles may copy per parsed document (each calibration owns a copy): a real /// database copies about a megabyte. const MAX_PROFILE_COPY_BYTES: usize = 32 << 20; +/// What one copied name costs besides its bytes: the `String` slot in its `Vec` and the allocator's header and rounding. +const NAME_ALLOCATION: usize = std::mem::size_of::() + 32; /// Largest radial error (half-diagonal units) accepted when converting a distortion to the DNG polynomial. pub const MAX_WARP_ERROR: f64 = 0.001; /// Largest relative error accepted when converting vignetting to the DNG polynomial. @@ -394,55 +396,66 @@ fn lens(n: &xml::Node, copy_budget: &mut usize) -> Vec { let aspect = aspect_of(n.kid_text("aspect-ratio")).unwrap_or(1.5); let rectilinear = n.kid_text("type").is_none_or(|t| t.eq_ignore_ascii_case("rectilinear")); let mounts = n.kids("mount").map(|m| m.text.trim().to_string()).filter(|m| !m.is_empty()).collect::>(); - // every calibration owns a copy of the lens's names: calibrations past the document's shared copy budget are left out - let meta_bytes = maker.len() + models.iter().chain(&mounts).map(|m| m.len() + 1).sum::() + 1; - let take = n.kids("calibration").count().min(*copy_budget / meta_bytes); - *copy_budget -= take * meta_bytes; - n.kids("calibration") - .take(take) - .map(|cal| { - let mut p = Profile { - maker: maker.to_string(), - models: models.clone(), - mounts: mounts.clone(), - crop_factor: crop_of(cal.attr("cropfactor")).unwrap_or(crop), - aspect_ratio: aspect_of(cal.attr("aspect-ratio")).unwrap_or(aspect), - rectilinear, - distortion: Vec::new(), - tca: Vec::new(), - vignetting: Vec::new(), - }; - for c in &cal.children { - let Some(focal) = positive(c.num("focal")) else { continue }; - let k = |name: &str| c.num(name).unwrap_or(0.0); - match (c.name.as_str(), c.attr("model")) { - ("distortion", Some("poly3")) => p.distortion.push((focal, Distortion::Poly3 { k1: k("k1") })), - ("distortion", Some("poly5")) => p.distortion.push((focal, Distortion::Poly5 { k1: k("k1"), k2: k("k2") })), - ("distortion", Some("ptlens")) => p.distortion.push((focal, Distortion::PtLens { a: k("a"), b: k("b"), c: k("c") })), - ("tca", Some("linear")) => { - let one = |name: &str| c.num(name).unwrap_or(1.0); - p.tca.push((focal, Tca::Linear { kr: one("kr"), kb: one("kb") })); - } - ("tca", Some("poly3")) => { - let one = |name: &str| c.num(name).unwrap_or(1.0); - p.tca.push((focal, Tca::Poly3 { vr: one("vr"), vb: one("vb"), cr: k("cr"), cb: k("cb"), br: k("br"), bb: k("bb") })); - } - ("vignetting", Some("pa")) => { - let (Some(aperture), distance) = (positive(c.num("aperture")), positive(c.num("distance")).unwrap_or(1000.0)) else { - continue; - }; - let model = Vignetting { k1: k("k1"), k2: k("k2"), k3: k("k3") }; - p.vignetting.push(VignettingPoint { focal, aperture, distance, model }); - } - _ => {} + // every kept calibration owns a copy of the lens's names, charged to the document's shared copy budget (the + // strings' own allocations count, not only their bytes): when it can't cover another copy, the rest are left out + let meta_bytes = std::mem::size_of::() + + NAME_ALLOCATION + + maker.len() + + models.iter().chain(&mounts).map(|m| m.len() + NAME_ALLOCATION).sum::(); + let mut out = Vec::new(); + for cal in n.kids("calibration") { + if *copy_budget < meta_bytes { + break; + } + let mut p = Profile { + maker: String::new(), + models: Vec::new(), + mounts: Vec::new(), + crop_factor: crop_of(cal.attr("cropfactor")).unwrap_or(crop), + aspect_ratio: aspect_of(cal.attr("aspect-ratio")).unwrap_or(aspect), + rectilinear, + distortion: Vec::new(), + tca: Vec::new(), + vignetting: Vec::new(), + }; + for c in &cal.children { + let Some(focal) = positive(c.num("focal")) else { continue }; + let k = |name: &str| c.num(name).unwrap_or(0.0); + match (c.name.as_str(), c.attr("model")) { + ("distortion", Some("poly3")) => p.distortion.push((focal, Distortion::Poly3 { k1: k("k1") })), + ("distortion", Some("poly5")) => p.distortion.push((focal, Distortion::Poly5 { k1: k("k1"), k2: k("k2") })), + ("distortion", Some("ptlens")) => p.distortion.push((focal, Distortion::PtLens { a: k("a"), b: k("b"), c: k("c") })), + ("tca", Some("linear")) => { + let one = |name: &str| c.num(name).unwrap_or(1.0); + p.tca.push((focal, Tca::Linear { kr: one("kr"), kb: one("kb") })); } + ("tca", Some("poly3")) => { + let one = |name: &str| c.num(name).unwrap_or(1.0); + p.tca.push((focal, Tca::Poly3 { vr: one("vr"), vb: one("vb"), cr: k("cr"), cb: k("cb"), br: k("br"), bb: k("bb") })); + } + ("vignetting", Some("pa")) => { + let (Some(aperture), distance) = (positive(c.num("aperture")), positive(c.num("distance")).unwrap_or(1000.0)) else { + continue; + }; + let model = Vignetting { k1: k("k1"), k2: k("k2"), k3: k("k3") }; + p.vignetting.push(VignettingPoint { focal, aperture, distance, model }); + } + _ => {} } - p.distortion.sort_by(|a, b| a.0.total_cmp(&b.0)); - p.tca.sort_by(|a, b| a.0.total_cmp(&b.0)); - p - }) - .filter(|p| !p.distortion.is_empty() || !p.tca.is_empty() || !p.vignetting.is_empty()) - .collect() + } + // a calibration with nothing to apply is dropped before its names are copied or charged + if p.distortion.is_empty() && p.tca.is_empty() && p.vignetting.is_empty() { + continue; + } + *copy_budget -= meta_bytes; + p.maker = maker.to_string(); + p.models = models.clone(); + p.mounts = mounts.clone(); + p.distortion.sort_by(|a, b| a.0.total_cmp(&b.0)); + p.tca.sort_by(|a, b| a.0.total_cmp(&b.0)); + out.push(p); + } + out } /// Linear interpolation weights in a sorted key list: the bracketing indices and the weight of the upper one. diff --git a/crates/lensdb/src/tests.rs b/crates/lensdb/src/tests.rs index 033111de7f..e535330a6e 100644 --- a/crates/lensdb/src/tests.rs +++ b/crates/lensdb/src/tests.rs @@ -383,6 +383,29 @@ fn calibrations_past_the_shared_copy_budget_are_left_out() { assert!(db.lenses.len() * 400_000 <= MAX_PROFILE_COPY_BYTES); } +/// Unusable calibrations (nothing to apply) are not copied or charged, so they can't use up the +/// budget a later usable one of the same lens needs. +#[test] +fn unusable_calibrations_do_not_spend_the_copy_budget() { + let name = "x".repeat(8 << 20); + let empty = ""; + let good = ""; + let xml = format!("M{name}{}{good}", empty.repeat(4)); + let db = LensDb::parse(&xml).unwrap(); + assert_eq!(db.lenses.len(), 1, "the usable calibration after the empty ones is kept"); +} + +/// Many tiny names cost their allocations, not only their bytes. +#[test] +fn many_tiny_names_are_charged_their_allocations() { + let models = "a".repeat(400_000); + let good = ""; + let xml = format!("M{models}{good}{good}{good}{good}"); + let db = LensDb::parse(&xml).unwrap(); + let names: usize = db.lenses.iter().map(|p| p.models.len()).sum(); + assert!(db.lenses.len() < 4 && names * std::mem::size_of::() <= MAX_PROFILE_COPY_BYTES, "{} profiles", db.lenses.len()); +} + #[cfg(unix)] #[test] fn load_dir_refuses_a_non_regular_xml_entry() { From 1c1eec51796b421d84e8a61ce663b334fdf56ccb Mon Sep 17 00:00:00 2001 From: FredApps Date: Wed, 4 Nov 2026 11:58:39 +0100 Subject: [PATCH 3/3] security(lensdb): refuse a non-regular *.xml before opening it (opening a FIFO blocks) The pathname is checked before File::open as well as the opened file after it: File::open on a FIFO blocks until a writer appears, so the post-open check alone never ran for one. Adds a unix regression test that hung before the fix. --- crates/lensdb/src/lib.rs | 6 +++++- crates/lensdb/src/tests.rs | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/lensdb/src/lib.rs b/crates/lensdb/src/lib.rs index 91f6723c1c..e34e07f862 100644 --- a/crates/lensdb/src/lib.rs +++ b/crates/lensdb/src/lib.rs @@ -280,7 +280,11 @@ fn exif_focal_range(exif: &[String]) -> Option<(f64, f64)> { fn read_capped(path: &Path) -> Result { use std::io::Read; let io = |e: std::io::Error| Error::Io(e.to_string()); - // the opened file is checked, not the pathname: a link or FIFO swapped in after a pathname check is refused too + // the pathname is checked before opening (opening a FIFO for reading blocks until a writer appears), then the + // opened file again: a link or FIFO swapped in after the pathname check is refused too + if !std::fs::metadata(path).map_err(io)?.is_file() { + return Err(Error::Io(format!("{}: not a regular file", path.display()))); + } let file = std::fs::File::open(path).map_err(io)?; let meta = file.metadata().map_err(io)?; if !meta.is_file() { diff --git a/crates/lensdb/src/tests.rs b/crates/lensdb/src/tests.rs index e535330a6e..9a25b53b1e 100644 --- a/crates/lensdb/src/tests.rs +++ b/crates/lensdb/src/tests.rs @@ -417,3 +417,21 @@ fn load_dir_refuses_a_non_regular_xml_entry() { assert!(db.is_empty()); assert_eq!(errs.len(), 1, "{errs:?}"); } + +/// A FIFO named `*.xml` is refused without opening it: opening a FIFO for reading blocks until a writer appears. +#[cfg(unix)] +#[test] +fn load_dir_refuses_a_fifo_without_blocking() { + let dir = std::env::temp_dir().join(format!("lensdb-test-fifo-{}", std::process::id())); + std::fs::create_dir(&dir).unwrap(); + let fifo = dir.join("f.xml"); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()); + if !made { + std::fs::remove_dir_all(&dir).unwrap(); + return; + } + let (db, errs) = LensDb::load_dir(&dir).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert!(db.is_empty()); + assert_eq!(errs.len(), 1, "{errs:?}"); +}