mirror of
https://github.com/storytold/photocraft.git
synced 2026-10-24 09:57:08 -06:00
Check deep EXR tile count tables against max_alloc (#5632)
* Check deep EXR tile count tables against max_alloc
A deep tile's sample-count table was decompressed at the header's full tile size with no
budget check, so a 20 KB file could build a 4 MiB table and a scratch copy under a 1 MiB
max_alloc. Try that size only when max_alloc has room for both; otherwise read the clipped size.
Fixes #4958.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* Keep deep EXR decompression within the size it was admitted at
The count-table check admits the table and its scratch copy, 2x the table, but Vec doubling
gave a 4 MiB ZIPS table 8 MiB of capacity (a 3 MiB RLE table 4 MiB), so an admitted table
and its copy could take 3x. Grow the decompressed buffer by doubling capped at the expected
size, inflating straight into it, and allocate it and the scratch copy fallibly.
Tests: decompressed buffers never reserve past the expected size; a near-maximal tile with a
clipped table decodes under every budget; per-tile admission; half and uint channels after
full-size tables; the 8 MiB boundary for uncompressed tables too.
Refs #4958.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* Tests: show deep tile tables are admitted one tile at a time
The 17x17 image in 16x16 tiles had an interior tile, which never reaches the max_alloc check,
so only three edge tables (6 KiB) were checked against the 8000-byte budget and a check that
summed admissions across tiles still passed. Use a 48x1 image instead: its three tiles are all
edge tiles, and under 4096 bytes a summed check fails at the third tile.
Refs #4958.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* Complete deep EXR count-table allocation admission
Account for scratch space on clipped and interior compressed tables while preserving raw-table budgets. Borrow packed input before admission and make initial RLE reservations and raw copies fallible.
Add single-channel admission boundaries, raw fallback controls, and hostile-size borrowed-read regressions.
* ci: run deep EXR regressions independently
* Drop the separate deep EXR CI job
It was added so these tests could run while main's workspace suite was
failing. Main is green again, and the regular test jobs run them.
This reverts commit 5aa4869b45.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: jonathnwxh <258340642+jonathanwxh-cell@users.noreply.github.com>
This commit is contained in:
jonathanwxh-cell
co-authored by
Claude Opus 5.5
jonathnwxh
Claude Opus 5.5
jonathnwxh
parent
64520f645e
commit
e4b2868a02
@@ -23,7 +23,7 @@ use half::f16;
|
||||
use crate::Format;
|
||||
use crate::error::CodecError;
|
||||
use crate::image::{ChannelLayout, DecodeWarning, DeepChannel, DeepImage, Image, SampleType};
|
||||
use crate::options::Limits;
|
||||
use crate::options::{Limits, alloc_zeroed};
|
||||
|
||||
const F: Format = Format::OpenExr;
|
||||
|
||||
@@ -39,9 +39,15 @@ fn unsup(reason: impl Into<String>) -> CodecError {
|
||||
// Every length and offset comes from the file, so every read is bounds-checked and a short
|
||||
// read is a malformed file, never a panic.
|
||||
|
||||
fn read_exact(cur: &mut Cursor<&[u8]>, n: usize, what: &str) -> Result<Vec<u8>, CodecError> {
|
||||
let mut v = vec![0u8; n];
|
||||
cur.read_exact(&mut v).map_err(|_| err(format!("truncated file while reading {what}")))?;
|
||||
fn read_exact<'a>(cur: &mut Cursor<&'a [u8]>, n: usize, what: &str) -> Result<&'a [u8], CodecError> {
|
||||
// Borrow packed payloads from the input. Copying them here allocates before count-table
|
||||
// admission and needlessly duplicates the table even when the second pass just skips it.
|
||||
let truncated = || err(format!("truncated file while reading {what}"));
|
||||
let start = usize::try_from(cur.position()).map_err(|_| truncated())?;
|
||||
let end = start.checked_add(n).ok_or_else(truncated)?;
|
||||
let bytes = *cur.get_ref();
|
||||
let v = bytes.get(start..end).ok_or_else(truncated)?;
|
||||
cur.set_position(end as u64);
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
@@ -81,10 +87,11 @@ fn differences_to_samples(buf: &mut [u8]) {
|
||||
}
|
||||
|
||||
/// Undo the ZIP half-split: the stream is `[first half][second half]` with the second half
|
||||
/// holding every second byte of the original; weave them back together.
|
||||
fn interleave_byte_blocks(buf: &mut [u8]) {
|
||||
/// holding every second byte of the original; weave them back together. The scratch copy is
|
||||
/// allocated fallibly, so a buffer that passed `max_alloc` but not the allocator is an error.
|
||||
fn interleave_byte_blocks(buf: &mut [u8]) -> Result<(), CodecError> {
|
||||
let half = buf.len().div_ceil(2);
|
||||
let mut out = vec![0u8; buf.len()];
|
||||
let mut out = alloc_zeroed::<u8>(buf.len())?;
|
||||
let mut k = 0usize;
|
||||
for i in 0..buf.len() / 2 {
|
||||
let (a, b) = (buf.get(i).copied(), buf.get(half + i).copied());
|
||||
@@ -105,12 +112,29 @@ fn interleave_byte_blocks(buf: &mut [u8]) {
|
||||
}
|
||||
}
|
||||
buf.copy_from_slice(&out);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Make room in `out` for `extra` more bytes, `limit` in all. Grows by doubling like `Vec`,
|
||||
/// but never reserves past `limit`: callers admit exactly that size against `max_alloc`, and
|
||||
/// plain doubling reserved up to twice as much. Fallible, so a failed allocation is an error.
|
||||
fn reserve_up_to(out: &mut Vec<u8>, extra: usize, limit: usize, what: &str) -> Result<(), CodecError> {
|
||||
let need = out.len().saturating_add(extra);
|
||||
if need > limit {
|
||||
return Err(err(format!("{what}: data expands past {limit} bytes")));
|
||||
}
|
||||
if need > out.capacity() {
|
||||
let target = out.capacity().saturating_mul(2).max(need).min(limit);
|
||||
out.try_reserve_exact(target.saturating_sub(out.len())).map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {target} bytes for {what}")))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// OpenEXR RLE tokens: a negative byte `n` means `-n` literal bytes follow, a non-negative
|
||||
/// byte `n` means the next byte repeats `n + 1` times (as `ImfRle.cpp` and the `exr` crate).
|
||||
fn rle_decode(src: &[u8], expected: usize, what: &str) -> Result<Vec<u8>, CodecError> {
|
||||
let mut out = Vec::with_capacity(expected.min(1 << 20));
|
||||
let mut out = Vec::new();
|
||||
reserve_up_to(&mut out, expected.min(1 << 20), expected, what)?;
|
||||
let mut i = 0;
|
||||
while i < src.len() {
|
||||
let count = src[i] as i8;
|
||||
@@ -119,6 +143,7 @@ fn rle_decode(src: &[u8], expected: usize, what: &str) -> Result<Vec<u8>, CodecE
|
||||
if out.len().saturating_add(n) > expected {
|
||||
return Err(err(format!("{what}: RLE data expands past {expected} bytes")));
|
||||
}
|
||||
reserve_up_to(&mut out, n, expected, what)?;
|
||||
if count < 0 {
|
||||
let end = i.saturating_add(n);
|
||||
let run = src.get(i..end.min(src.len())).ok_or_else(|| err(format!("{what}: RLE run points past the compressed data")))?;
|
||||
@@ -143,7 +168,10 @@ fn rle_decode(src: &[u8], expected: usize, what: &str) -> Result<Vec<u8>, CodecE
|
||||
/// has the uncompressed length is stored raw whatever the declared compression.
|
||||
fn decompress(compression: Compression, packed: &[u8], expected: usize, what: &str) -> Result<Vec<u8>, CodecError> {
|
||||
if packed.len() == expected {
|
||||
return Ok(packed.to_vec());
|
||||
let mut out = Vec::new();
|
||||
reserve_up_to(&mut out, expected, expected, what)?;
|
||||
out.extend_from_slice(packed);
|
||||
return Ok(out);
|
||||
}
|
||||
match compression {
|
||||
Compression::Uncompressed => Err(err(format!("{what}: expected {expected} uncompressed bytes, found {}", packed.len()))),
|
||||
@@ -151,20 +179,38 @@ fn decompress(compression: Compression, packed: &[u8], expected: usize, what: &s
|
||||
// Like ZIPS, RLE runs over the predicted, byte-split stream.
|
||||
let mut out = rle_decode(packed, expected, what)?;
|
||||
differences_to_samples(&mut out);
|
||||
interleave_byte_blocks(&mut out);
|
||||
interleave_byte_blocks(&mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
Compression::ZIP1 => {
|
||||
let mut out = Vec::with_capacity(expected.min(1 << 20));
|
||||
flate2::read::ZlibDecoder::new(packed)
|
||||
.take(expected as u64 + 1)
|
||||
.read_to_end(&mut out)
|
||||
.map_err(|_| err(format!("{what}: zlib data is malformed")))?;
|
||||
// Inflate straight into `out`, grown with `reserve_up_to` rather than by
|
||||
// `read_to_end`, whose doubling reserves past `expected`. Once `expected` bytes are
|
||||
// in, a one-byte read checks that the stream ends there.
|
||||
let malformed = || err(format!("{what}: zlib data is malformed"));
|
||||
let mut zlib = flate2::read::ZlibDecoder::new(packed);
|
||||
let mut out = Vec::new();
|
||||
let mut filled = 0;
|
||||
while filled < expected {
|
||||
if filled == out.len() {
|
||||
let grow = out.len().max(1 << 13).min(expected - filled);
|
||||
reserve_up_to(&mut out, grow, expected, what)?;
|
||||
out.resize(filled + grow, 0);
|
||||
}
|
||||
let n = zlib.read(out.get_mut(filled..).unwrap_or_default()).map_err(|_| malformed())?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
filled += n;
|
||||
}
|
||||
out.truncate(filled);
|
||||
if filled == expected && zlib.read(&mut [0u8; 1]).map_err(|_| malformed())? != 0 {
|
||||
return Err(err(format!("{what}: zlib data expands past {expected} bytes")));
|
||||
}
|
||||
if out.len() != expected {
|
||||
return Err(err(format!("{what}: zlib data expands to {} bytes, expected {expected}", out.len())));
|
||||
}
|
||||
differences_to_samples(&mut out);
|
||||
interleave_byte_blocks(&mut out);
|
||||
interleave_byte_blocks(&mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
other => Err(unsup(format!("deep data with {other:?} compression (NONE, RLE and ZIPS are supported)"))),
|
||||
@@ -340,13 +386,41 @@ pub(crate) fn decode_deep(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Res
|
||||
if packed_table > remaining || packed_data > remaining.saturating_sub(packed_table) {
|
||||
return Err(err("deep chunk sizes point past the end of the file"));
|
||||
}
|
||||
let table_bytes = read_exact(&mut cur, packed_table as usize, "the sample-count table")?;
|
||||
let what = "the sample-count table";
|
||||
let table_bytes = read_exact(&mut cur, packed_table as usize, what)?;
|
||||
let expected_table = block_pixels.len().checked_mul(4).ok_or_else(|| err("count table overflows"))?;
|
||||
// A compressed clipped table also needs its scratch copy. The data-window check only
|
||||
// budgets four bytes per channel per pixel, insufficient for a one-channel image.
|
||||
// An equal-sized payload takes decompress's raw path, which allocates no scratch.
|
||||
let buffers = if table_bytes.len() == expected_table { 1 } else { 2 };
|
||||
let clipped_need = (expected_table as u64).saturating_mul(buffers);
|
||||
if clipped_need > limits.max_alloc {
|
||||
return Err(CodecError::LimitExceeded(format!(
|
||||
"{what} needs {clipped_need} bytes including decompression scratch, exceeding max_alloc {}",
|
||||
limits.max_alloc
|
||||
)));
|
||||
}
|
||||
// An edge tile's table may still be stored at the full tile size (its pixels first, in
|
||||
// rows of the clipped width, then unused entries), as the OpenEXR library writes it.
|
||||
let full_table = tile_w.saturating_mul(tile_h).saturating_mul(4).max(expected_table);
|
||||
let raw = decompress(compression, &table_bytes, full_table, "the sample-count table")
|
||||
.or_else(|_| decompress(compression, &table_bytes, expected_table, "the sample-count table"))?;
|
||||
// That size comes from the header's tile size alone, which nothing ties to the data
|
||||
// window, so it is tried only when `max_alloc` has room for the table and for the copy
|
||||
// that undoes its byte split. Otherwise only the clipped size is read.
|
||||
let full_table = tile_w.checked_mul(tile_h).and_then(|n| n.checked_mul(4));
|
||||
let raw = match full_table {
|
||||
// Scan lines and interior tiles: the full size is the clipped size.
|
||||
Some(full) if full <= expected_table => decompress(compression, table_bytes, expected_table, what)?,
|
||||
Some(full) if (full as u64).saturating_mul(2) <= limits.max_alloc => {
|
||||
decompress(compression, table_bytes, full, what).or_else(|_| decompress(compression, table_bytes, expected_table, what))?
|
||||
}
|
||||
_ => decompress(compression, table_bytes, expected_table, what).map_err(|_| {
|
||||
// Four bytes per pixel of the tile, twice: the table and its scratch copy.
|
||||
let need = (tile_w as u64).saturating_mul(tile_h as u64).saturating_mul(8);
|
||||
CodecError::LimitExceeded(format!(
|
||||
"{what} of a {tile_w}x{tile_h} deep tile does not decode at the clipped size, and at the full tile size it needs {need} bytes with its scratch copy, exceeding max_alloc {}",
|
||||
limits.max_alloc
|
||||
))
|
||||
})?,
|
||||
};
|
||||
|
||||
// The table is cumulative within each row of the block (a tile restarts at every row);
|
||||
// `prefix` is cumulative over the whole block, and each pixel's own count is the difference.
|
||||
@@ -423,7 +497,7 @@ pub(crate) fn decode_deep(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Res
|
||||
// The count table sits between the sizes and the sample data; skip its bytes.
|
||||
read_exact(&mut cur, packed_table as usize, "the sample-count table")?;
|
||||
let data_bytes = read_exact(&mut cur, chunk.packed_data as usize, "the deep sample data")?;
|
||||
let raw = decompress(compression, &data_bytes, chunk.unpacked_data as usize, "the deep sample data")?;
|
||||
let raw = decompress(compression, data_bytes, chunk.unpacked_data as usize, "the deep sample data")?;
|
||||
// Row by row (a scan line block is one row), and within a row channel by channel, as
|
||||
// flat tiles are laid out: each pixel's samples are contiguous within its channel.
|
||||
let mut in_off = 0usize;
|
||||
@@ -611,3 +685,82 @@ pub(crate) fn flatten(deep: &DeepImage) -> Result<Image, CodecError> {
|
||||
img.warnings.push(DecodeWarning::DeepFlattened { format: F, max_samples_per_pixel: max_samples });
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write as _;
|
||||
|
||||
#[test]
|
||||
fn packed_reads_borrow_input_and_reject_bad_sizes_before_allocation() {
|
||||
let bytes = [1u8, 2, 3, 4];
|
||||
let mut cur = Cursor::new(bytes.as_slice());
|
||||
let prefix = read_exact(&mut cur, 3, "x").expect("three input bytes");
|
||||
assert_eq!(prefix, &[1, 2, 3]);
|
||||
assert_eq!(prefix.as_ptr(), bytes.as_ptr());
|
||||
assert_eq!(cur.position(), 3);
|
||||
assert!(matches!(read_exact(&mut cur, usize::MAX, "x"), Err(CodecError::Malformed { .. })));
|
||||
assert_eq!(cur.position(), 3);
|
||||
assert_eq!(read_exact(&mut cur, 1, "x").expect("last byte"), &[4]);
|
||||
assert!(matches!(read_exact(&mut cur, 1, "x"), Err(CodecError::Malformed { .. })));
|
||||
cur.set_position(u64::MAX);
|
||||
assert!(matches!(read_exact(&mut cur, 1, "x"), Err(CodecError::Malformed { .. })));
|
||||
}
|
||||
|
||||
/// What the writer stores before compressing: the byte split, then the +128 predictor.
|
||||
fn predicted(raw: &[u8]) -> Vec<u8> {
|
||||
let half = raw.len().div_ceil(2);
|
||||
let mut out = vec![0u8; raw.len()];
|
||||
for (i, &v) in raw.iter().enumerate() {
|
||||
out[if i % 2 == 0 { i / 2 } else { half + i / 2 }] = v;
|
||||
}
|
||||
let mut prev = 0i16;
|
||||
for (i, b) in out.iter_mut().enumerate() {
|
||||
let v = i16::from(*b);
|
||||
if i > 0 {
|
||||
*b = (v - prev + 128) as u8;
|
||||
}
|
||||
prev = v;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// #4958: callers admit `expected` bytes for a decompressed buffer (twice that for a count
|
||||
/// table, with the copy that undoes its byte split), so decompression must not reserve
|
||||
/// more. `Vec`'s doubling gave a 4 MiB ZIPS table 8 MiB of capacity and a 3 MiB RLE table
|
||||
/// 4 MiB, so the "admitted" table and its copy took up to three times the table size.
|
||||
#[test]
|
||||
fn decompressed_buffers_reserve_no_more_than_expected() {
|
||||
for n in [1usize << 20, 2 << 20, (3 << 20) + 1, 4 << 20, 5 << 20] {
|
||||
let raw = vec![0u8; n];
|
||||
let p = predicted(&raw);
|
||||
let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast());
|
||||
z.write_all(&p).expect("write to a Vec");
|
||||
let zips = z.finish().expect("finish a Vec");
|
||||
// RLE: the first byte as a literal, then runs of up to 128 copies of the next.
|
||||
let mut rle = vec![0xFF, p[0]];
|
||||
for run in p[1..].chunks(128) {
|
||||
rle.extend_from_slice(&[(run.len() - 1) as u8, run[0]]);
|
||||
}
|
||||
for (compression, packed) in [(Compression::ZIP1, &zips), (Compression::RLE, &rle)] {
|
||||
let out = decompress(compression, packed, n, "the table").expect("decompresses");
|
||||
assert_eq!(out, raw, "{compression:?}, {n} bytes");
|
||||
assert!(out.capacity() <= n, "{compression:?}: a {n} byte buffer reserved {} bytes", out.capacity());
|
||||
}
|
||||
// One byte short, the same data expands past the expected size and is refused.
|
||||
for (compression, packed) in [(Compression::ZIP1, &zips), (Compression::RLE, &rle)] {
|
||||
assert!(decompress(compression, packed, n - 1, "the table").is_err(), "{compression:?}, {n} bytes");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A size past the limit is malformed data, and a reservation the allocator refuses is an
|
||||
/// error rather than an abort.
|
||||
#[test]
|
||||
fn reservations_past_the_limit_or_the_allocator_are_errors() {
|
||||
let mut v = Vec::new();
|
||||
assert!(matches!(reserve_up_to(&mut v, 2, 1, "x"), Err(CodecError::Malformed { .. })));
|
||||
assert!(matches!(reserve_up_to(&mut v, isize::MAX as usize + 1, usize::MAX, "x"), Err(CodecError::LimitExceeded(_))));
|
||||
assert_eq!(v.capacity(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +118,23 @@ fn gen_deep(
|
||||
tile: Option<(usize, usize)>,
|
||||
counts: &[u32],
|
||||
sample: &dyn Fn(usize, usize) -> Vec<f32>,
|
||||
) -> Vec<u8> {
|
||||
gen_deep_tables(w, h, channels, compression, tile, counts, sample, false)
|
||||
}
|
||||
|
||||
/// [`gen_deep`]; with `full_tile_tables` every tile's count table is stored at the full tile
|
||||
/// size, as the OpenEXR library writes edge tiles: the block's entries first (in rows of its
|
||||
/// clipped width), then zeros.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn gen_deep_tables(
|
||||
w: usize,
|
||||
h: usize,
|
||||
channels: &[Chan],
|
||||
compression: u8,
|
||||
tile: Option<(usize, usize)>,
|
||||
counts: &[u32],
|
||||
sample: &dyn Fn(usize, usize) -> Vec<f32>,
|
||||
full_tile_tables: bool,
|
||||
) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&0x0131_2f76u32.to_le_bytes());
|
||||
@@ -216,6 +233,10 @@ fn gen_deep(
|
||||
table.extend_from_slice(&acc.to_le_bytes());
|
||||
}
|
||||
}
|
||||
assert_eq!(table.len(), table_len(b));
|
||||
if let (true, Some((tw, th))) = (full_tile_tables, tile) {
|
||||
table.resize(tw * th * 4, 0);
|
||||
}
|
||||
let mut data = Vec::new();
|
||||
for row in b.pixels.chunks(b.row_w) {
|
||||
for (ci, ch) in channels.iter().enumerate() {
|
||||
@@ -237,7 +258,6 @@ fn gen_deep(
|
||||
c.extend_from_slice(&(data.len() as u64).to_le_bytes());
|
||||
c.extend_from_slice(&ct);
|
||||
c.extend_from_slice(&cd);
|
||||
assert_eq!(table.len(), table_len(b));
|
||||
assert_eq!(data.len(), data_len(b));
|
||||
chunks.push((bi, c));
|
||||
}
|
||||
@@ -608,6 +628,136 @@ fn hostile_sizes_and_offsets_error() {
|
||||
assert!(matches!(decode_deep_exr(&far, &Limits::none()), Err(CodecError::Malformed { .. })));
|
||||
}
|
||||
|
||||
/// #4958: a deep tile's count table may be stored at the full tile size, and that size comes
|
||||
/// from the header alone: nothing ties it to the data window. One 1024×1024 tile over a 4×4
|
||||
/// image, its table stored at that size, makes a compressed file of tens of KB whose table
|
||||
/// inflates to 4 MiB, plus as much again for the copy that undoes the byte split. Under a
|
||||
/// budget smaller than both the table must be refused before it is allocated; with room for
|
||||
/// both it decodes. Tables stored at the clipped size, and the full-size edge tables of
|
||||
/// ordinary tiles, still decode under a small budget.
|
||||
#[test]
|
||||
fn deep_tile_count_table_is_admitted_against_max_alloc() {
|
||||
let budget = |max_alloc: u64| Limits { max_alloc, ..Limits::none() };
|
||||
let tight = budget(1 << 20);
|
||||
let table: u64 = 1024 * 1024 * 4; // the full tile's count table, in bytes
|
||||
let scan = decode_deep_exr(&gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba), &Limits::none()).expect("scan lines");
|
||||
for compression in [0u8, 1, 2] {
|
||||
let big = gen_deep_tables(4, 4, &RGBAZ, compression, Some((1024, 1024)), COUNTS_ONE, &sample_rgba, true);
|
||||
assert_eq!(decode_deep_exr(&big, &Limits::none()).expect("full-size table, no limits"), scan, "compression {compression}");
|
||||
let got = decode_deep_exr(&big, &tight);
|
||||
assert!(
|
||||
matches!(got, Err(CodecError::LimitExceeded(_))),
|
||||
"compression {compression}: a 4 MiB count table under a 1 MiB budget gave {:?}",
|
||||
got.map(|d| d.total_samples())
|
||||
);
|
||||
let flat = photocraft_codecs::decode_with(&big, &photocraft_codecs::DecodeOptions { limits: tight, ..Default::default() });
|
||||
assert!(matches!(flat, Err(CodecError::LimitExceeded(_))), "compression {compression}: the flat decode gave {:?}", flat.map(|i| i.width()));
|
||||
// The same tile with its table stored at the clipped size fits the small budget.
|
||||
let clipped = gen_deep(4, 4, &RGBAZ, compression, Some((1024, 1024)), COUNTS_ONE, &sample_rgba);
|
||||
assert_eq!(decode_deep_exr(&clipped, &tight).expect("clipped table, 1 MiB budget"), scan, "compression {compression}");
|
||||
if compression != 0 {
|
||||
assert!(big.len() < 1 << 20, "compression {compression}: the file is {} bytes", big.len());
|
||||
}
|
||||
// The table and its scratch copy are admitted together: 8 MiB.
|
||||
assert_eq!(decode_deep_exr(&big, &budget(2 * table)).expect("room for the table and its copy"), scan, "compression {compression}");
|
||||
let got = decode_deep_exr(&big, &budget(2 * table - 1));
|
||||
assert!(matches!(got, Err(CodecError::LimitExceeded(_))), "compression {compression}: one byte short gave {:?}", got.map(|d| d.total_samples()));
|
||||
}
|
||||
// Ordinary tiles: full-size edge tables decode under the small budget, like clipped ones.
|
||||
let counts: Vec<u32> = (0..35).map(|p| (p * 3) % 4).collect();
|
||||
let sample = |p: usize, ci: usize| -> Vec<f32> { (0..counts[p]).map(|k| [0.5, 0.25, 0.125 * k as f32, 1.0, 10.0 - k as f32][ci]).collect() };
|
||||
for compression in [0u8, 1, 2] {
|
||||
let full = gen_deep_tables(7, 5, &RGBAZ, compression, Some((3, 2)), &counts, &sample, true);
|
||||
let clipped = gen_deep(7, 5, &RGBAZ, compression, Some((3, 2)), &counts, &sample);
|
||||
assert_eq!(
|
||||
decode_deep_exr(&full, &tight).expect("full-size edge tables"),
|
||||
decode_deep_exr(&clipped, &tight).expect("clipped edge tables"),
|
||||
"compression {compression}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// #4958, the edges of that admission. A tile of the largest size exr accepts (2^30 - 2 a side,
|
||||
/// a 2^62 byte count table) decodes from a clipped table under every budget, including no limits,
|
||||
/// where its full size passes the check and is only tried as the data arrives. Each tile is
|
||||
/// admitted on its own, because its table is freed before the next chunk: three 16×16 tiles over
|
||||
/// a 48×1 image are all edge tiles, so every full-size table reaches the check (an interior tile's
|
||||
/// full size equals its clipped size, so it uses the clipped check). They need 2 KiB each for
|
||||
/// the table and its copy, 6 KiB in all, and decode under a budget of 4096 bytes. Half and uint
|
||||
/// channels decode after full-size tables like float ones.
|
||||
#[test]
|
||||
fn deep_tile_table_admission_edges() {
|
||||
let budget = |max_alloc: u64| Limits { max_alloc, ..Limits::none() };
|
||||
let scan = decode_deep_exr(&gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba), &Limits::none()).expect("scan lines");
|
||||
let side = (1 << 30) - 2;
|
||||
for compression in [0u8, 1, 2] {
|
||||
let bytes = gen_deep(4, 4, &RGBAZ, compression, Some((side, side)), COUNTS_ONE, &sample_rgba);
|
||||
for limits in [Limits::none(), Limits::default(), budget(1 << 20)] {
|
||||
assert_eq!(decode_deep_exr(&bytes, &limits).expect("the largest tile, clipped table"), scan, "compression {compression}, {limits:?}");
|
||||
}
|
||||
}
|
||||
let counts: Vec<u32> = (0..48).map(|p| (p % 3) as u32).collect();
|
||||
let half: [Chan; 5] = RGBAZ.map(|c| Chan { ty: 1, ..c });
|
||||
let [a, b, g, r, z] = RGBAZ;
|
||||
let with_id = [a, b, g, r, z, Chan { name: "id", ty: 0 }];
|
||||
for channels in [&RGBAZ[..], &half[..], &with_id[..]] {
|
||||
let sample = |p: usize, ci: usize| -> Vec<f32> { (0..counts[p]).map(|k| [0.5, 0.25, 0.125, 1.0, 2.0 + k as f32, 7.0][ci]).collect() };
|
||||
for compression in [0u8, 1, 2] {
|
||||
let full = gen_deep_tables(48, 1, channels, compression, Some((16, 16)), &counts, &sample, true);
|
||||
let clipped = gen_deep(48, 1, channels, compression, Some((16, 16)), &counts, &sample);
|
||||
let what = format!("{} channels, compression {compression}", channels.len());
|
||||
let want = decode_deep_exr(&clipped, &Limits::none()).expect("clipped tables");
|
||||
assert_eq!(decode_deep_exr(&full, &budget(4096)).expect("full-size tables, admitted one at a time"), want, "{what}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A one-channel data-window check admits only four bytes per pixel, but a compressed count
|
||||
/// table and its scratch copy need eight. Cover scanlines, interior tiles and the clipped fallback;
|
||||
/// the existing multichannel cases have enough pixel allowance to hide this gap.
|
||||
#[test]
|
||||
fn clipped_count_tables_include_scratch_in_the_budget() {
|
||||
let channels = [Chan { name: "Y", ty: 2 }];
|
||||
let counts = vec![0u32; 64];
|
||||
let sample = |_: usize, _: usize| Vec::new();
|
||||
let budget = |max_alloc: u64| Limits { max_alloc, ..Limits::none() };
|
||||
for tile in [None, Some((64, 1)), Some((128, 2))] {
|
||||
for compression in [0u8, 1, 2] {
|
||||
let bytes = gen_deep(64, 1, &channels, compression, tile, &counts, &sample);
|
||||
let what = format!("tile {tile:?}, compression {compression}");
|
||||
let want = decode_deep_exr(&bytes, &Limits::none()).expect("unlimited decode");
|
||||
let required = if compression == 0 { 256 } else { 512 };
|
||||
assert_eq!(decode_deep_exr(&bytes, &budget(required)).expect("table and any scratch fit exactly"), want, "{what}");
|
||||
assert!(matches!(decode_deep_exr(&bytes, &budget(required - 1)), Err(CodecError::LimitExceeded(_))), "{what}");
|
||||
let flat = photocraft_codecs::decode_with(&bytes, &photocraft_codecs::DecodeOptions { limits: budget(required - 1), ..Default::default() });
|
||||
assert!(matches!(flat, Err(CodecError::LimitExceeded(_))), "{what}");
|
||||
if compression == 0 {
|
||||
assert_eq!(decode_deep_exr(&bytes, &budget(511)).expect("raw table needs no scratch"), want, "{what}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RLE and ZIPS store incompressible tables raw too. Their actual payload, not the header's
|
||||
/// compression tag, decides whether a scratch buffer is needed.
|
||||
#[test]
|
||||
fn raw_count_tables_need_no_scratch_even_with_compression() {
|
||||
let channels = [Chan { name: "Y", ty: 2 }];
|
||||
let limits = Limits { max_alloc: 4, ..Limits::none() };
|
||||
for tile in [None, Some((1, 1)), Some((2, 2))] {
|
||||
for compression in [0u8, 1, 2] {
|
||||
let bytes = gen_deep(1, 1, &channels, compression, tile, &[0], &|_, _| Vec::new());
|
||||
let header_end = header_end_of(&bytes);
|
||||
let chunk = u64::from_le_bytes(bytes[header_end..header_end + 8].try_into().expect("chunk offset")) as usize;
|
||||
let table_size = chunk + if tile.is_some() { 16 } else { 4 };
|
||||
let packed = u64::from_le_bytes(bytes[table_size..table_size + 8].try_into().expect("table size"));
|
||||
assert_eq!(packed, 4, "the four-byte count table is stored raw");
|
||||
let want = decode_deep_exr(&bytes, &Limits::none()).expect("unlimited decode");
|
||||
assert_eq!(decode_deep_exr(&bytes, &limits).expect("raw table fits"), want, "tile {tile:?}, compression {compression}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the attributes like the format prescribes: the offset table starts after the
|
||||
/// header's terminating zero byte.
|
||||
fn header_end_of(bytes: &[u8]) -> usize {
|
||||
|
||||
Reference in New Issue
Block a user