mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-26 11:09:23 -06:00
security/raw: bound hostile-memory paths in raw decoders
Sparse walkers, truncated MEF/lossy-JPEG strips, embedded JPEG sizes, the CR3 preview marker walk, OpcodeList3 full-image passes and caller-supplied Deflate tile edges each let a small file or one bad argument multiply allocations or CPU work thousands of times. Each path now is bounded, every fix with a regression test that failed (or blew up) before it, plus a tiff::image single_chunk helper so preview extraction never builds a descriptor vector for a hostile multi-strip layout.
This commit is contained in:
Dylan M. Taylor
@@ -94,6 +94,17 @@ pub const SOFTWARE: &str = "LightCraft DNG 2";
|
||||
/// (3.9 s vs 0.55 s on 4 cores) to save 3 % of the file (266 vs 275 MB); 5–12 % on raw and rendered DNGs.
|
||||
const DEFLATE_LEVEL: u8 = 1;
|
||||
|
||||
/// Largest Deflate tile edge the writer pads to (`tile` is caller-supplied in [`DngCompression::Deflate`]):
|
||||
/// square-tile buffers grow with the square, so an unclamped edge lets a request pad a tiny valid image into
|
||||
/// a multi-gigabyte buffer and billions of edge-replicate iterations. 2048² samples (≤ 24 MB of f32) is far
|
||||
/// above the 256 default and every real tile this writer makes.
|
||||
const MAX_DEFLATE_TILE: u32 = 2048;
|
||||
|
||||
/// `tile` aligned down to a multiple of 16 and clamped to [`MAX_DEFLATE_TILE`].
|
||||
fn deflate_tile(tile: u32) -> usize {
|
||||
(tile.max(16) & !15).min(MAX_DEFLATE_TILE) as usize
|
||||
}
|
||||
|
||||
/// IEEE 754 binary32 → binary16 (round to nearest even; overflow → ±inf, NaN kept).
|
||||
pub fn f32_to_f16(v: f32) -> u16 {
|
||||
let b = v.to_bits();
|
||||
@@ -318,7 +329,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
|
||||
|
||||
match (&raw.data, opts.compression) {
|
||||
(RawData::F32(v), DngCompression::Deflate { tile, half }) => {
|
||||
let tile = (tile.max(16) & !15) as usize;
|
||||
let tile = deflate_tile(tile);
|
||||
let bp = if half { 2 } else { 4 };
|
||||
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![(bp * 8) as u16; cpp]));
|
||||
ifd.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp]));
|
||||
@@ -417,7 +428,7 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
|
||||
ifd.set_image(ImageData::Strips { rows_per_strip: rows as u32, strips });
|
||||
}
|
||||
DngCompression::Deflate { tile, .. } => {
|
||||
let tile = (tile.max(16) & !15) as usize;
|
||||
let tile = deflate_tile(tile);
|
||||
ifd.set(t::BITS_PER_SAMPLE, Value::Short(vec![16; cpp]));
|
||||
ifd.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE]));
|
||||
ifd.set(t::PREDICTOR, Value::Short(vec![2]));
|
||||
@@ -546,3 +557,45 @@ pub fn write_rendered_dng(
|
||||
let order = ByteOrder::Little;
|
||||
write_dng(&raw, &DngWriteOptions { compression: DngCompression::Deflate { tile: 256, half: true }, order, xmp })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{BlackLevel, OpcodeLists, RawData, RawFormat, RawImage, Rect};
|
||||
|
||||
/// Deflate tiles of `tile × tile` pad into a square buffer that grows with the square: a
|
||||
/// caller-supplied edge of `u32::MAX` would make the writer pad an 8 × 8 image into billions of
|
||||
/// edge-replicate iterations and a multi-gigabyte buffer. The clamp keeps the bounded shape — same
|
||||
/// file as requesting the cap exactly, and a DNG either way.
|
||||
#[test]
|
||||
fn a_hostile_deflate_tile_edge_is_clamped() {
|
||||
let (w, h) = (8usize, 8usize);
|
||||
let full = Rect::new(0, 0, w, h);
|
||||
let raw = RawImage {
|
||||
format: RawFormat::Dng,
|
||||
width: w,
|
||||
height: h,
|
||||
cpp: 1,
|
||||
data: RawData::U16((0..64).map(|i| i as u16 * 129).collect()),
|
||||
cfa: None,
|
||||
bits: 12,
|
||||
black: BlackLevel::uniform(0.0),
|
||||
white: vec![4095.0],
|
||||
active_area: full,
|
||||
crop: full,
|
||||
orientation: lightcraft_geom::Orientation::Normal,
|
||||
color: crate::ColorData::default(),
|
||||
wb_multipliers: None,
|
||||
linearized: false,
|
||||
pixel_aspect: 1.0,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata: Default::default(),
|
||||
lens_metadata: Default::default(),
|
||||
};
|
||||
let opts = |tile: u32| DngWriteOptions { compression: DngCompression::Deflate { tile, half: false }, ..DngWriteOptions::default() };
|
||||
let hostile = write_dng(&raw, &opts(u32::MAX)).expect("the clamped tile writes");
|
||||
assert_eq!(hostile, write_dng(&raw, &opts(2048)).expect("the cap itself writes"), "u32::MAX is clamped to the cap");
|
||||
assert_ne!(hostile, write_dng(&raw, &opts(256)).expect("a normal tile writes"), "the clamp is not a no-op");
|
||||
assert_eq!(crate::probe(&hostile), Some(RawFormat::Dng), "the clamped output is still a DNG");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -656,18 +656,38 @@ fn fix_pixels(buf: &mut [f32], w: usize, h: usize, cpp: usize, cfa: Option<&Cfa>
|
||||
}
|
||||
}
|
||||
|
||||
/// Most full-image passes of one `OpcodeList3`: every warp record is a full-image allocation and resample
|
||||
/// pass and every bad-pixel record a full flatten-and-rebuild, and a hostile record may repeat either
|
||||
/// thousands of times. Real DNGs carry at most a couple; passes past the cap are skipped.
|
||||
const MAX_LIST3_PASSES: usize = 8;
|
||||
|
||||
/// Apply `OpcodeList3` (on the demosaiced RGB image, values normalised to [0, 1]).
|
||||
pub fn apply_list3(list: &[Opcode], img: &mut Rgb32f) {
|
||||
if list.is_empty() {
|
||||
return;
|
||||
}
|
||||
let (w, h) = (img.width, img.height);
|
||||
let mut passes = 0usize;
|
||||
for op in applied_ops(list) {
|
||||
match op {
|
||||
Opcode::WarpRectilinear { planes, center } => *img = warp_rectilinear(img, planes, *center),
|
||||
Opcode::WarpRectilinear2 { planes, center, reciprocal, .. } => *img = warp_rectilinear2(img, planes, *center, *reciprocal),
|
||||
Opcode::WarpRectilinear { planes, center } => {
|
||||
if passes < MAX_LIST3_PASSES {
|
||||
passes += 1;
|
||||
*img = warp_rectilinear(img, planes, *center);
|
||||
}
|
||||
}
|
||||
Opcode::WarpRectilinear2 { planes, center, reciprocal, .. } => {
|
||||
if passes < MAX_LIST3_PASSES {
|
||||
passes += 1;
|
||||
*img = warp_rectilinear2(img, planes, *center, *reciprocal);
|
||||
}
|
||||
}
|
||||
Opcode::WarpFisheye { .. } | Opcode::TrimBounds { .. } | Opcode::Unknown { .. } => {}
|
||||
other => {
|
||||
if passes >= MAX_LIST3_PASSES {
|
||||
continue;
|
||||
}
|
||||
passes += 1;
|
||||
let mut flat: Vec<f32> = img.data.iter().flat_map(|p| *p).collect();
|
||||
apply_list(std::slice::from_ref(other), &mut flat, w, h, 3, None, 1.0);
|
||||
for (d, c) in img.data.iter_mut().zip(flat.as_chunks::<3>().0) {
|
||||
@@ -765,6 +785,27 @@ pub(crate) mod tests {
|
||||
a.iter().flat_map(|v| v.to_be_bytes()).collect()
|
||||
}
|
||||
|
||||
/// Warp and bad-pixel records each cost a full-image pass, and a hostile `OpcodeList3` repeats them
|
||||
/// thousands of times ([`MAX_LIST3_PASSES`] bounds the multiplier); identity warps below the cap must
|
||||
/// still apply, exactly, so the same image comes out.
|
||||
#[test]
|
||||
fn list3_pass_cap_skips_repeated_records() {
|
||||
let mut img = Rgb32f::new(8, 6);
|
||||
for (i, px) in img.data.iter_mut().enumerate() {
|
||||
*px = [i as f32 / 2.0, (i % 8) as f32, (i / 8) as f32];
|
||||
}
|
||||
let orig: Vec<[f32; 3]> = img.data.clone();
|
||||
let list: Vec<Opcode> =
|
||||
(0..MAX_LIST3_PASSES * 5).map(|_| Opcode::WarpRectilinear { planes: vec![[1.0, 0.0, 0.0, 0.0, 0.0, 0.0]], center: [0.5, 0.5] }).collect();
|
||||
apply_list3(&list, &mut img);
|
||||
assert_eq!(img.data, orig, "identity warps change nothing, and the cap skips the repeated rest");
|
||||
|
||||
// a real warp below the cap is still applied
|
||||
let one = [Opcode::WarpRectilinear { planes: vec![[2.0, 0.0, 0.0, 0.0, 0.0, 0.0]], center: [0.5, 0.5] }];
|
||||
apply_list3(&one, &mut img);
|
||||
assert_ne!(img.data, orig, "a genuine warp record is not skipped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_and_apply_area_ops() {
|
||||
let (w, h) = (4usize, 3usize);
|
||||
|
||||
+62
-11
@@ -103,8 +103,6 @@ fn jpeg_frame(b: &[u8]) -> Option<(u32, u32, u8)> {
|
||||
let be16 = |k: usize| Some(u32::from(u16::from_be_bytes([at(k)?, at(k + 1)?])));
|
||||
Some((be16(7)?, be16(5)?, at(9)?))
|
||||
}
|
||||
|
||||
/// Whether `b` is a JPEG XL file: a bare codestream or the ISO-BMFF container's signature box.
|
||||
fn is_jxl(b: &[u8]) -> bool {
|
||||
b.starts_with(&[0xff, 0x0a]) || b.starts_with(&[0, 0, 0, 0x0c, b'J', b'X', b'L', b' ', 0x0d, 0x0a, 0x87, 0x0a])
|
||||
}
|
||||
@@ -128,20 +126,17 @@ fn candidates<'a>(data: &'a [u8], ifd: &Ifd, base: u64, out: &mut Vec<&'a [u8]>)
|
||||
}
|
||||
if matches!(ifd.u16(t::COMPRESSION), Some(6) | Some(7) | Some(34892))
|
||||
&& let Ok(info) = ifd.image()
|
||||
&& let Some(chunk) = info.single_chunk(data.len() as u64)
|
||||
&& let Some(s) = chunk_bytes(data, &chunk)
|
||||
{
|
||||
let chunks = info.chunks(data.len() as u64);
|
||||
if chunks.len() == 1
|
||||
&& let Some(s) = chunk_bytes(data, &chunks[0])
|
||||
{
|
||||
out.push(s);
|
||||
}
|
||||
out.push(s);
|
||||
}
|
||||
// a rendered (RGB or grey, never CFA/LinearRaw/mask) JPEG XL preview in one chunk
|
||||
if ifd.u16(t::COMPRESSION) == Some(t::compression::JPEG_XL)
|
||||
&& matches!(ifd.u16(t::PHOTOMETRIC), Some(t::photometric::BLACK_IS_ZERO | t::photometric::RGB))
|
||||
&& let Ok(info) = ifd.image()
|
||||
&& let [chunk] = info.chunks(data.len() as u64).as_slice()
|
||||
&& let Some(s) = chunk_bytes(data, chunk)
|
||||
&& let Some(chunk) = info.single_chunk(data.len() as u64)
|
||||
&& let Some(s) = chunk_bytes(data, &chunk)
|
||||
&& is_jxl(s)
|
||||
{
|
||||
out.push(s);
|
||||
@@ -397,9 +392,15 @@ fn cr3_preview(bytes: &[u8]) -> Option<&[u8]> {
|
||||
/// Canon CR3 (ISO base media file): the `PRVW` box (Laurent Clévy's CR3 notes; layout confirmed on a CC0
|
||||
/// sample) is `u32 size, "PRVW", u32 0, u16 ?, u16 width, u16 height, u16 ?, u32 jpeg length, JPEG`; the smaller
|
||||
/// `THMB` box has the same shape. Returns the larger valid one.
|
||||
/// Marker-walk budget of the CR3 fallback: candidates can overlap (each claiming the rest of the file) and
|
||||
/// [`is_dct_jpeg`] walks each marker by marker, so walking every candidate would be quadratic in file size.
|
||||
/// The `THMB`/`PRVW` previews this finds are far below the cap together; once it is spent the scan stops.
|
||||
const CR3_MAX_WALK: usize = 8 << 20;
|
||||
|
||||
fn cr3_preview_boxes(bytes: &[u8]) -> Option<&[u8]> {
|
||||
let mut best: Option<&[u8]> = None;
|
||||
for tag in [b"PRVW", b"THMB"] {
|
||||
let mut walked = 0usize;
|
||||
'tags: for tag in [b"PRVW", b"THMB"] {
|
||||
let mut from = 0;
|
||||
while let Some(i) = bytes.get(from..).and_then(|s| s.windows(4).position(|w| w == tag)).map(|p| p + from) {
|
||||
from = i + 4;
|
||||
@@ -411,6 +412,10 @@ fn cr3_preview_boxes(bytes: &[u8]) -> Option<&[u8]> {
|
||||
continue;
|
||||
}
|
||||
let jpeg = &bytes[j..j + len];
|
||||
if walked + jpeg.len() > CR3_MAX_WALK {
|
||||
break 'tags;
|
||||
}
|
||||
walked += jpeg.len();
|
||||
if is_dct_jpeg(jpeg) {
|
||||
if best.is_none_or(|b| b.len() < jpeg.len()) {
|
||||
best = Some(jpeg);
|
||||
@@ -440,6 +445,52 @@ mod tests {
|
||||
j
|
||||
}
|
||||
|
||||
/// Preview extraction only serves single-chunk JPEG-compressed IFDs and asks the `single_chunk` first:
|
||||
/// a TIFF whose strip array declares 100 000 one-row strips and no StripByteCounts must not build their
|
||||
/// descriptor vector, and of course finds no preview in it.
|
||||
#[test]
|
||||
fn a_multi_strip_jpeg_ifd_costs_no_descriptor_vector() {
|
||||
let mut ifd = IfdBuilder::new();
|
||||
ifd.set(t::COMPRESSION, Value::Short(vec![7]));
|
||||
ifd.set(t::IMAGE_WIDTH, Value::Long(vec![16]));
|
||||
ifd.set(t::IMAGE_LENGTH, Value::Long(vec![100_000]));
|
||||
ifd.set(t::ROWS_PER_STRIP, Value::Short(vec![1]));
|
||||
ifd.set(t::STRIP_OFFSETS, Value::Byte((0..100_000).map(|i| (i % 255 + 1) as u8).collect()));
|
||||
let file = TiffWriter::default().write(&[ifd]).unwrap();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || tx.send(embedded_preview(&file)));
|
||||
assert_eq!(rx.recv_timeout(std::time::Duration::from_secs(2)), Ok(None));
|
||||
}
|
||||
|
||||
/// CR3 candidates claim overlapping extents: each block's first bytes double as a 40-byte jump marker so
|
||||
/// a walk jogs past the next header and reaches the end of the file, making every one of the 9000
|
||||
/// candidates walk its whole suffix before the marker-walk budget — quadratic without it. Nothing here
|
||||
/// is a JPEG (no frame header), so the answer is `None` either way.
|
||||
#[test]
|
||||
fn cr3_preview_boxes_stop_walking_when_the_budget_is_spent() {
|
||||
// one 256-byte block per candidate, all marker-aligned so walks chain through later blocks:
|
||||
// 0..4, size (itself a jump marker of 40 bytes); 4..8, "PRVW"; 20..24, the claimed length
|
||||
// (to the file end); 24..28, an SOI plus a marker jumping to the fill at 40; 64..256, a fill of
|
||||
// 4-byte markers (`ff 01 00 02`).
|
||||
let blocks = 9000usize;
|
||||
let total = blocks * 256;
|
||||
let mut f = Vec::with_capacity(total);
|
||||
for k in 0..blocks {
|
||||
let base = k * 256;
|
||||
f.extend_from_slice(&[0xff, 0x01, 0x00, 0x26, b'P', b'R', b'V', b'W', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
f.extend_from_slice(&u32::try_from(total - base - 24).unwrap().to_be_bytes());
|
||||
f.extend_from_slice(&[0xff, 0xd8, 0x00, 0x24, 0xff, 0x01, 0x00, 0x26, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
f.resize(base + 64, 0); // 40..64 filler: walks jump over it, so its bytes are free
|
||||
for _ in 0..48 {
|
||||
f.extend_from_slice(&[0xff, 0x01, 0x00, 0x02]);
|
||||
}
|
||||
}
|
||||
assert_eq!(f.len(), total);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || tx.send(cr3_preview_boxes(&f).map(|s| s.len())));
|
||||
assert_eq!(rx.recv_timeout(std::time::Duration::from_secs(2)), Ok(None), "no PRVW candidate is a JPEG");
|
||||
}
|
||||
|
||||
/// A minimal RAF: the fixed header (JPEG offset/length at 84/88), the JPEG, then a raw tail the
|
||||
/// caller must not have to read. `jpeg_len` is written as declared, so a test can claim more than
|
||||
/// the JPEG holds the way a padded container does.
|
||||
|
||||
@@ -35,6 +35,12 @@ const MAX_ENTRIES: u64 = 8192;
|
||||
const MAX_BOXES: usize = 4096;
|
||||
/// Child IFDs listed by one `SubIFDs` entry.
|
||||
const MAX_CHILDREN: u64 = 4096;
|
||||
/// Pending child-IFD offsets the walker may queue at once: no more than it can ever visit
|
||||
/// (the same [`MAX_IFDS`] the seen set is held to, deduplicated as they are popped), so an alley
|
||||
/// of hostile `SubIFDs` entries can't grow the queue into the tens of millions of pointers.
|
||||
/// The plan never guesses: the walk refuses and the plan reports the whole file when it would go
|
||||
/// past the cap.
|
||||
const MAX_PENDING: usize = MAX_IFDS;
|
||||
/// The RAF raw container's pointer to its raw IFD.
|
||||
const RAF_RAW_IFD: u16 = 0xf000;
|
||||
|
||||
@@ -308,6 +314,10 @@ fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ra
|
||||
};
|
||||
let mut stack = vec![first];
|
||||
let mut seen = HashSet::new();
|
||||
// Every IFD table and out-of-line value, collected and merged once at the end: an insert
|
||||
// per entry would scan the ranges of every entry before it — quadratic in the metadata
|
||||
// this walker collects.
|
||||
let mut spans: Vec<(u64, u64)> = Vec::new();
|
||||
while let Some(off) = stack.pop() {
|
||||
if off == 0 || off >= len || seen.len() >= MAX_IFDS || !seen.insert(off) {
|
||||
continue;
|
||||
@@ -329,7 +339,7 @@ fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ra
|
||||
if !w.want(at(off), at(end)) {
|
||||
continue;
|
||||
}
|
||||
meta.insert(at(off), at(end));
|
||||
spans.push((at(off), at(end)));
|
||||
for i in 0..n_read {
|
||||
let p = table + i * entry;
|
||||
let Ok(pu) = usize::try_from(p) else { continue };
|
||||
@@ -348,7 +358,7 @@ fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ra
|
||||
};
|
||||
let Some(value_end) = start.checked_add(total).filter(|&e| e <= len) else { continue };
|
||||
if total > inline {
|
||||
meta.insert(at(start), at(value_end));
|
||||
spans.push((at(start), at(value_end)));
|
||||
}
|
||||
let child = matches!(tag, t::SUB_IFDS | t::EXIF_IFD | t::GPS_IFD | t::INTEROP_IFD) || extra.contains(&tag);
|
||||
let locates = child || matches!(tag, t::STRIP_OFFSETS | t::STRIP_BYTE_COUNTS | t::TILE_OFFSETS | t::TILE_BYTE_COUNTS);
|
||||
@@ -359,14 +369,21 @@ fn walk_tiff(w: &mut Want, origin: u64, data: &[u8], extra: &[u16]) -> Result<Ra
|
||||
let take = if tag == t::SUB_IFDS { count.min(MAX_CHILDREN) } else { count.min(1) };
|
||||
for k in 0..take {
|
||||
if let Some(v) = unsigned(data, order, typ, start + k * typ.size() as u64) {
|
||||
if stack.len() >= MAX_PENDING {
|
||||
return Err("too many child IFDs queued to visit");
|
||||
}
|
||||
stack.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
if complete && let Some(next) = offset_at(table_end) {
|
||||
if stack.len() >= MAX_PENDING {
|
||||
return Err("too many child IFDs queued to visit");
|
||||
}
|
||||
stack.push(next);
|
||||
}
|
||||
}
|
||||
meta.extend(spans);
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
@@ -585,8 +602,74 @@ mod tests {
|
||||
assert_eq!(run(&file, 16, 0).0, HeaderPlan::Whole("an ARW6 header probe reads its tiles"));
|
||||
}
|
||||
|
||||
/// A TIFF with a million one-byte strips listed in descending order and all byte counts 0: the plan is
|
||||
/// built in one sort and merge, not one pass over all strips (or all holes) per strip.
|
||||
/// An IFD whose `SubIFDs` entries would queue 33 million child pointers (8192 entries of
|
||||
/// 4096 values apiece, all pointing at the same array) is refused at the walker's pending
|
||||
/// cap instead of growing the queue — and the plan reports the whole file, never guessing.
|
||||
#[test]
|
||||
fn a_subifds_alley_is_refused_at_the_pending_cap() {
|
||||
// First read everything, so the walk reaches the directory table itself.
|
||||
let n = 8192u32;
|
||||
let len = 8 + 2 + (n * 12 + 4 + 4 * 4096) as usize;
|
||||
let mut tiff = b"II*\0\x08\0\0\0".to_vec();
|
||||
tiff.extend_from_slice(&(n as u16).to_le_bytes());
|
||||
let mut entry = |tag: u16, typ: u16, count: u32, value: u32| {
|
||||
tiff.extend_from_slice(&tag.to_le_bytes());
|
||||
tiff.extend_from_slice(&typ.to_le_bytes());
|
||||
tiff.extend_from_slice(&count.to_le_bytes());
|
||||
tiff.extend_from_slice(&value.to_le_bytes());
|
||||
};
|
||||
// header (8) + count field (2) + the {n} numbers and pointer (4), then the value array
|
||||
let values_at = (8 + 2u64 + u64::from(n) * 12 + 4) as u32;
|
||||
for _ in 0..n {
|
||||
entry(t::SUB_IFDS, 4, MAX_CHILDREN as u32, values_at);
|
||||
}
|
||||
tiff.extend_from_slice(&0u32.to_le_bytes());
|
||||
tiff.extend((1..=4096u32).flat_map(u32::to_le_bytes));
|
||||
let mut loaded = Ranges::new();
|
||||
loaded.insert(0, len as u64);
|
||||
let mut w = Want::new(&tiff, &loaded);
|
||||
assert_eq!(walk_tiff(&mut w, 0, &tiff, &[]), Err("too many child IFDs queued to visit"));
|
||||
}
|
||||
|
||||
/// A TIFF with many out-of-line values across 512 directories is planned in one sort and
|
||||
/// merge, not one pass over all accumulated ranges per value (CodeRabbit #52: before, each
|
||||
/// `Ranges::insert` copied every range found before it — the ~million disjoint values of the
|
||||
/// fixture below ask for more copying than the timeout allows).
|
||||
#[test]
|
||||
fn many_out_of_line_values_plan_fast() {
|
||||
let (ifds, values) = (512usize, 2048usize);
|
||||
let ifd_len = 2 + values * 12 + 4;
|
||||
let values_at = 8 + ifds * ifd_len;
|
||||
let len = values_at + values * 8;
|
||||
let mut tiff = vec![0u8; len];
|
||||
tiff[..8].copy_from_slice(b"II*\0\x08\0\0\0");
|
||||
for i in 0..ifds {
|
||||
let at = 8 + i * ifd_len;
|
||||
// {values} out-of-line strings (count 8 > the 4 inline bytes), then the next-IFD
|
||||
// pointer; every value is a disjoint 8-byte range for the merge to handle.
|
||||
tiff[at..at + 2].copy_from_slice(&(values as u16).to_le_bytes());
|
||||
for k in 0..values {
|
||||
let e = at + 2 + k * 12;
|
||||
tiff[e..e + 2].copy_from_slice(&t::MAKE.to_le_bytes());
|
||||
tiff[e + 2..e + 4].copy_from_slice(&2u16.to_le_bytes()); // ASCII: 8 bytes > inline — a span per entry
|
||||
tiff[e + 4..e + 8].copy_from_slice(&8u32.to_le_bytes());
|
||||
tiff[e + 8..e + 12].copy_from_slice(&(values_at as u32 + (k * 8) as u32).to_le_bytes());
|
||||
}
|
||||
let next = if i == ifds - 1 { 0u32 } else { (at + ifd_len) as u32 };
|
||||
tiff[at + ifd_len - 4..at + ifd_len].copy_from_slice(&next.to_le_bytes());
|
||||
}
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut loaded = Ranges::new();
|
||||
loaded.insert(0, len as u64);
|
||||
let _ = tx.send(header_plan(&tiff, &loaded));
|
||||
});
|
||||
rx.recv_timeout(std::time::Duration::from_secs(20)).expect("header_plan took longer than 20 s");
|
||||
}
|
||||
|
||||
/// A TIFF with a million one-byte strips listed in descending order and all byte counts 0:
|
||||
/// the plan is built in one sort and merge, not one pass over all strips (or all holes) per
|
||||
/// strip.
|
||||
#[test]
|
||||
fn many_strips_without_byte_counts_plan_fast() {
|
||||
let n = 1_000_000u32;
|
||||
|
||||
@@ -179,7 +179,7 @@ fn decode_chunk(data: &[u8], info: &ImageInfo, c: &Chunk, order: ByteOrder, pack
|
||||
}
|
||||
comp::LOSSY_JPEG => {
|
||||
// lossy DNG: each chunk is a baseline (DCT) JPEG of 8-bit samples
|
||||
let (px, jw, jh, jc) = lossy_jpeg(src, cpp)?;
|
||||
let (px, jw, jh, jc) = lossy_jpeg(src, cpp, MAX_JPEG_SAMPLES)?;
|
||||
let mut v = vec![0u16; n];
|
||||
for y in 0..ch.min(jh) {
|
||||
for x in 0..cw.min(jw) {
|
||||
@@ -197,7 +197,23 @@ fn decode_chunk(data: &[u8], info: &ImageInfo, c: &Chunk, order: ByteOrder, pack
|
||||
// the gate caps at MAX_SAMPLES (≤ 2^30), so the `usize` cast below is lossless on 32-bit targets too
|
||||
let inflate_len =
|
||||
crate::checked_pixel_count(row_bytes, ch as u64).ok_or_else(|| RawError::Corrupt("bad TIFF chunk dimensions".into()))?;
|
||||
// Deflate compresses at most about 1:1032, so a source this far under half of its
|
||||
// nominal length cannot reconstruct the chunk: refused before the nominal buffer is
|
||||
// inflated and predicted, which padding-only work would make gigabytes.
|
||||
if (src.len() as u64).saturating_mul(2048) < inflate_len {
|
||||
return Err(RawError::Corrupt(format!("a deflated chunk of {inflate_len} bytes is packed into only {} bytes", src.len())));
|
||||
}
|
||||
let raw = inflate(src, inflate_len as usize)?;
|
||||
// The stream has to reach the last row inside the image (edge strips may stop there);
|
||||
// it is laid out in nominal rows, so shorter is a truncated chunk, not a picture.
|
||||
let last = if float { vw as u64 * cpp as u64 * bytes_per as u64 } else { (vw as u64 * cpp as u64 * bits as u64).div_ceil(8) };
|
||||
let visible = row_bytes.saturating_mul(vh.saturating_sub(1) as u64).saturating_add(last);
|
||||
if (raw.len() as u64) < visible {
|
||||
return Err(RawError::Corrupt(format!(
|
||||
"a deflated chunk holds {} bytes, fewer than the {visible} bytes its in-image samples span",
|
||||
raw.len()
|
||||
)));
|
||||
}
|
||||
unpack_chunk(&raw, order, packing, cw, ch, cpp, bits, float, info.predictor)
|
||||
}
|
||||
comp::JPEG_XL => jxl_chunk(src, cw, ch, vw, vh, cpp, bits, float),
|
||||
@@ -228,19 +244,31 @@ fn jxl_chunk_check(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp:
|
||||
jxl_chunk(src, cw, ch, vw, vh, cpp, bits, float).map(|_| ())
|
||||
}
|
||||
|
||||
/// The most samples a tiled JPEG chunk or a white-balance preview may decode to: far more than
|
||||
/// any real lossy tile or preview carries, and small enough that a JPEG header alone cannot size
|
||||
/// a multi-gigabyte allocation.
|
||||
pub(crate) const MAX_JPEG_SAMPLES: usize = 1 << 26;
|
||||
|
||||
/// Decode one baseline JPEG chunk to 8-bit samples: (samples, width, height, channels).
|
||||
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
|
||||
pub(crate) fn lossy_jpeg(src: &[u8], cpp: usize, max_samples: usize) -> Result<(Vec<u8>, usize, usize, usize)> {
|
||||
use zune_core::bytestream::ZCursor;
|
||||
use zune_core::colorspace::ColorSpace;
|
||||
use zune_core::options::DecoderOptions;
|
||||
let cs = if cpp == 1 { ColorSpace::Luma } else { ColorSpace::RGB };
|
||||
let opts = DecoderOptions::default().set_max_width(1 << 16).set_max_height(1 << 16).set_strict_mode(false).jpeg_set_out_colorspace(cs);
|
||||
let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(src), opts);
|
||||
let px = d.decode().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
|
||||
// The header is read first: the JPEG's own dimensions, not the enclosing tile's, size every
|
||||
// allocation `decode` makes, and they may claim billions of pixels.
|
||||
d.decode_headers().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
|
||||
let info = d.info().ok_or_else(|| RawError::Corrupt("lossy JPEG tile without a header".into()))?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let c = if cpp == 1 { 1 } else { 3 };
|
||||
if w == 0 || h == 0 || px.len() < w * h * c {
|
||||
let out = w as u64 * h as u64 * c as u64;
|
||||
if w == 0 || h == 0 || out > max_samples as u64 {
|
||||
return Err(RawError::Corrupt(format!("a lossy JPEG of {w} × {h} × {c} would exceed the {max_samples} output samples allowed")));
|
||||
}
|
||||
let px = d.decode().map_err(|e| RawError::Corrupt(format!("lossy JPEG tile: {e}")))?;
|
||||
if px.len() < w * h * c {
|
||||
return Err(RawError::Corrupt("lossy JPEG tile: short output".into()));
|
||||
}
|
||||
Ok((px, w, h, c))
|
||||
@@ -471,4 +499,57 @@ mod lossy_tests {
|
||||
// half of the visible rows are missing
|
||||
assert!(decode(8, 4).is_err());
|
||||
}
|
||||
|
||||
/// A lossy JPEG whose header alone claims more samples than any real tile or preview (here
|
||||
/// 40000 × 40000 × 3) is refused from the header, before the decoder allocates its output
|
||||
/// from the JPEG's own dimensions.
|
||||
#[test]
|
||||
fn an_oversized_lossy_jpeg_header_is_refused_before_decoding() {
|
||||
let mut j = vec![0xff, 0xd8]; // SOI
|
||||
// SOF0: precision 8, 40000 (0x9c40) x 40000, three components
|
||||
j.extend([0xff, 0xc0, 0x00, 0x11, 0x08, 0x9c, 0x40, 0x9c, 0x40, 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01]);
|
||||
// SOS with three components so header parsing completes normally, then EOI
|
||||
j.extend([0xff, 0xda, 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x11, 0x00, 0x03, 0x11, 0x00, 0x00, 0x3f, 0x00, 0x00, 0xff, 0xd9]);
|
||||
let e = lossy_jpeg(&j, 3, MAX_JPEG_SAMPLES).unwrap_err();
|
||||
assert!(matches!(e, RawError::Corrupt(_)), "{e:?}");
|
||||
// a header inside the budget passes the check (and then fails to decode only for lack of data)
|
||||
j[12..14].copy_from_slice(&1024u16.to_be_bytes());
|
||||
j[14..16].copy_from_slice(&1024u16.to_be_bytes());
|
||||
assert!(lossy_jpeg(&j, 3, MAX_JPEG_SAMPLES).is_ok() || matches!(lossy_jpeg(&j, 3, MAX_JPEG_SAMPLES), Err(RawError::Corrupt(_))));
|
||||
}
|
||||
|
||||
/// A deflated tile is refused when its packed bytes are too few for deflate to have produced
|
||||
/// the nominal chunk, or when the stream stops short of the rows inside the image; a fully
|
||||
/// covered flat tile still decodes.
|
||||
#[test]
|
||||
fn a_short_or_undercovering_deflated_tile_is_refused() {
|
||||
let info = |byte_counts: u64| ImageInfo {
|
||||
width: 512,
|
||||
height: 512,
|
||||
bits_per_sample: vec![16],
|
||||
samples_per_pixel: 1,
|
||||
compression: comp::ADOBE_DEFLATE,
|
||||
photometric: 32803,
|
||||
planar: 1,
|
||||
predictor: 1,
|
||||
sample_format: 1,
|
||||
new_subfile_type: 0,
|
||||
layout: Layout::Tiles { tile_width: 512, tile_height: 512 },
|
||||
offsets: vec![0],
|
||||
byte_counts: vec![byte_counts],
|
||||
};
|
||||
// a complete flat tile: 512 KiB of zeros compresses to a few hundred bytes (well within
|
||||
// deflate's packing bound) and inflates to every nominal byte the tile claims
|
||||
let zlib = miniz_oxide::deflate::compress_to_vec_zlib(&vec![0u8; 512 * 512 * 2], 6);
|
||||
assert_eq!(read_image(&zlib, &info(zlib.len() as u64), ByteOrder::Little, Packing::Msb), Ok(RawData::U16(vec![0; 512 * 512])));
|
||||
// the same 1 KiB packed, but the stream claims a whole tile: a few hundred bytes cannot
|
||||
// reconstruct 512 KiB from a decoder that compresses no further than about 1:1032
|
||||
let zlib = miniz_oxide::deflate::compress_to_vec_zlib(&vec![0u8; 1024], 6);
|
||||
assert!(matches!(read_image(&zlib, &info(512 * 512 * 2), ByteOrder::Little, Packing::Msb), Err(RawError::Corrupt(_))), "{zlib:?}");
|
||||
// 500 KiB of zeros compress to a stream within the packing bound and inflating to more
|
||||
// bytes than the in-image samples span, but 24 KiB short of the whole nominal tile:
|
||||
// refused as truncated rather than filled with the predictor's padding
|
||||
let zlib = miniz_oxide::deflate::compress_to_vec_zlib(&vec![0u8; 500 * 1000], 6);
|
||||
assert!(matches!(read_image(&zlib, &info(zlib.len() as u64), ByteOrder::Little, Packing::Msb), Err(RawError::Corrupt(_))));
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+58
-1
@@ -89,6 +89,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
// the white balance is measured from the samples, so the header probe reads them too
|
||||
let chunks = info.chunks(bytes.len() as u64);
|
||||
let src = chunks.first().and_then(|c| chunk_bytes(bytes, c)).ok_or_else(|| RawError::Corrupt("Mef strip outside file".into()))?;
|
||||
// 12-bit MSB packing takes 3 bytes per 2 samples; a strip that cannot hold the declared
|
||||
// samples is a truncated file, not a picture — unpacking it would run past the real bytes
|
||||
// through every output sample (a short strip asks for the whole allocation and more).
|
||||
let need = n.div_ceil(2).saturating_mul(3);
|
||||
if src.len() < need {
|
||||
return Err(RawError::Corrupt(format!("a Mef strip of {} bytes cannot hold the declared {n} samples", src.len())));
|
||||
}
|
||||
let mut d = vec![0u16; n];
|
||||
crate::unpack::unpack_msb(src, 12, &mut d);
|
||||
let white = super::white_from_data(&d, 12);
|
||||
@@ -138,6 +145,10 @@ mod tests {
|
||||
}
|
||||
|
||||
fn sample(w: u32, h: u32, px: &[u16]) -> Vec<u8> {
|
||||
sample_strip(w, h, &pack(px))
|
||||
}
|
||||
|
||||
fn sample_strip(w: u32, h: u32, strip: &[u8]) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![w]));
|
||||
@@ -148,7 +159,7 @@ mod tests {
|
||||
raw.set(t::ORIENTATION, Value::Short(vec![6]));
|
||||
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
|
||||
raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![pack(px)] });
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![strip.to_vec()] });
|
||||
let mut ifd0 = IfdBuilder::new();
|
||||
ifd0.set(t::MAKE, Value::Ascii("Mamiya-OP Co.,Ltd.".into()));
|
||||
ifd0.set(t::MODEL, Value::Ascii("MAMIYA ZD".into()));
|
||||
@@ -156,6 +167,52 @@ mod tests {
|
||||
TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
/// A strip that cannot hold the declared samples is a truncated file, refused as corrupt
|
||||
/// instead of unpacked: `unpack_msb` would otherwise run `n` samples through the few real
|
||||
/// bytes, filling the rest from nothing.
|
||||
#[test]
|
||||
fn a_strip_too_short_for_the_declared_samples_is_refused() {
|
||||
let px: Vec<u16> = (0..16 * 6).map(|i| (i * 41 % 4096) as u16).collect();
|
||||
let f = sample(16, 6, &px);
|
||||
assert!(crate::decode(&f).is_ok());
|
||||
// A truncated file shaped like a real one: IFD tables first, strip last — only its first
|
||||
// 20 packed bytes remain, while the IFD still declares the full 1.5 B/sample length. The
|
||||
// probe accepts the file on the declared length; before the byte gate below, `unpack_msb`
|
||||
// ran the dimension-sized allocation through those 20 bytes, filling it from nothing.
|
||||
let u32be = |b: &[u8], a: usize| u32::from_be_bytes([b[a], b[a + 1], b[a + 2], b[a + 3]]);
|
||||
let full = pack(&px);
|
||||
let at = f.windows(full.len()).position(|w| w == full).expect("strip bytes in the file");
|
||||
let after_strip = at + full.len();
|
||||
let moved = after_strip - (at + 20);
|
||||
let mut g = f[..8].to_vec();
|
||||
g.extend(std::iter::repeat_n(0u8, at + 20 - 8));
|
||||
g.extend_from_slice(&f[after_strip..]);
|
||||
// repoint the first-IFD pointer and ifd0's MAKE / MODEL / SubIfds offsets (the tables now
|
||||
// sit `moved` earlier); the raw sub-IFD's strip offset and byte count are patched below
|
||||
let poke = |b: &mut [u8], a: usize| {
|
||||
let v = u32::from_be_bytes([b[a], b[a + 1], b[a + 2], b[a + 3]]);
|
||||
if v as usize >= after_strip {
|
||||
b[a..a + 4].copy_from_slice(&(v - moved as u32).to_be_bytes());
|
||||
}
|
||||
};
|
||||
let ifd0_at = u32be(&g, 4) as usize - moved;
|
||||
g[4..8].copy_from_slice(&(ifd0_at as u32).to_be_bytes());
|
||||
let n = u16::from_be_bytes([g[ifd0_at], g[ifd0_at + 1]]) as usize;
|
||||
for i in 0..n {
|
||||
poke(&mut g, ifd0_at + 2 + i * 12 + 8);
|
||||
}
|
||||
// the raw sub-IFD: entries follow its count; tag 273 (StripOffsets) is the seventh, held
|
||||
// inline — repoint it at the strip's new home, and append the truncated strip for real
|
||||
let sub_at = u32be(&g, ifd0_at + 2 + 2 * 12 + 8) as usize;
|
||||
let e_offsets = sub_at + 2 + 6 * 12 + 8;
|
||||
let strip_at = g.len() as u32;
|
||||
g[e_offsets..e_offsets + 4].copy_from_slice(&strip_at.to_be_bytes());
|
||||
g.extend_from_slice(&full[..20]);
|
||||
assert_eq!(crate::probe(&g), Some(RawFormat::Mef));
|
||||
let r = crate::decode(&g);
|
||||
assert!(matches!(r, Err(RawError::Corrupt(_))), "{r:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_packed_12_bit() {
|
||||
let px: Vec<u16> = (0..16 * 6).map(|i| (i * 41 % 4096) as u16).collect();
|
||||
|
||||
@@ -66,7 +66,7 @@ fn median(v: &mut [f64]) -> Option<f64> {
|
||||
|
||||
fn from_preview(bytes: &[u8], f: &Frame) -> Option<[f32; 3]> {
|
||||
let jpeg = crate::preview::embedded_preview(bytes)?;
|
||||
let (px, pw, ph, ch) = crate::tiffraw::lossy_jpeg(&jpeg, 3).ok()?;
|
||||
let (px, pw, ph, ch) = crate::tiffraw::lossy_jpeg(&jpeg, 3, crate::tiffraw::MAX_JPEG_SAMPLES).ok()?;
|
||||
if ch != 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -118,14 +118,34 @@ impl ImageInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// All chunks that have an offset. Missing byte counts are inferred as "to the next offset / end of data"
|
||||
/// using `data_len`.
|
||||
pub fn chunks(&self, data_len: u64) -> Vec<Chunk> {
|
||||
/// The chunk count the image declares (offsets present, cut to the layout's plane capacity) with the
|
||||
/// geometry `chunks` needs to place each one. No descriptors are built, so a pathological strip array
|
||||
/// costs it a length check.
|
||||
fn chunk_grid(&self) -> (usize, u32, usize) {
|
||||
let (across, down) = self.grid();
|
||||
let per_plane = across as usize * down as usize;
|
||||
let planes = if self.planar == 2 { self.samples_per_pixel as usize } else { 1 };
|
||||
(self.offsets.len().min(per_plane.saturating_mul(planes)), across, per_plane)
|
||||
}
|
||||
|
||||
fn declared_chunk_count(&self) -> usize {
|
||||
self.chunk_grid().0
|
||||
}
|
||||
|
||||
/// The one chunk when the image has exactly one, else `None` — and no descriptor vector beyond it. Preview
|
||||
/// extraction only serves single-chunk images, so it asks here before serving a hostile multi-strip layout.
|
||||
pub fn single_chunk(&self, data_len: u64) -> Option<Chunk> {
|
||||
if self.declared_chunk_count() != 1 {
|
||||
return None;
|
||||
}
|
||||
self.chunks(data_len).into_iter().next()
|
||||
}
|
||||
|
||||
/// All chunks that have an offset. Missing byte counts are inferred as "to the next offset / end of data"
|
||||
/// using `data_len`.
|
||||
pub fn chunks(&self, data_len: u64) -> Vec<Chunk> {
|
||||
let (n, across, per_plane) = self.chunk_grid();
|
||||
let (cw, ch) = self.chunk_size();
|
||||
let n = self.offsets.len().min(per_plane.saturating_mul(planes));
|
||||
let mut out = Vec::with_capacity(n);
|
||||
// the distinct offsets in order, for chunks without a byte count (sorted once, not scanned per chunk)
|
||||
let mut sorted: Vec<u64> = Vec::new();
|
||||
@@ -192,6 +212,18 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// One offset is one chunk, and `single_chunk` refuses to build descriptors when the image declares more.
|
||||
#[test]
|
||||
fn single_chunk_is_the_only_chunk() {
|
||||
let one = strips(vec![50], vec![10]);
|
||||
assert_eq!(one.single_chunk(100).map(|c| (c.index, c.offset, c.len)), one.chunks(100).first().map(|c| (c.index, c.offset, c.len)));
|
||||
// the inferred length matches too, and never falls back past the data
|
||||
let inferred = strips(vec![70], vec![]);
|
||||
assert_eq!(inferred.single_chunk(100).map(|c| c.len), Some(30));
|
||||
assert!(strips(vec![50, 10], vec![10, 5]).single_chunk(100).is_none(), "a multi-chunk image has no single chunk");
|
||||
assert!(strips(Vec::new(), Vec::new()).single_chunk(100).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_byte_counts_run_to_the_next_larger_offset() {
|
||||
let info = strips(vec![50, 10, 30, 10, 70], vec![0, 0, 5]);
|
||||
|
||||
Reference in New Issue
Block a user