mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-10-31 13:37:35 -06:00
M11: redaction removes Group 3 on bytes whole, which the renderer may draw as other rows
PdfCraft's bootstrap renderer draws byte-aligned Group 3 whose fill ends each end-of-line code on a byte (as TIFF writes it) as other rows than decode_ccitt reads (black box, at every /K, with /EndOfLine or not: mostly the first row, then blank). Redaction cleared pixels in the picture decode_ccitt reads, so a mark over what PdfCraft showed needn't cover what was cleared, and content shown blank stayed in the copy. Such an image (/K 0 or more, /EncodedByteAlign) is now removed whole, as every CCITT image was before this PR; Group 3 off bytes and Group 4 are still cleared. The image's parameters come from Document::fax_params, the reading of a fax image's /DecodeParms entry that decoded_image already did, now on its own (it was to come with #4404, which uses it too). The redaction README, the module doc, the redaction and core.image-filters notes and the spec parity table's status line and revision row say so, and the core.image-filters note now records the renderer's misdrawing. group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared: a white 16 × 2 picture, checked to decode as such, is removed whole on bytes (no codes; codes at /K 0; at /K 1 with /EncodedByteAlign by reference) and cleared off bytes and as Group 4 on bytes. It fails without the guard; cargo-mutants catches every viable mutant of it. From reviews with the pdfcraft-review skill. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01P6yUtvWGPsJpMLyh6pxSXG
This commit is contained in:
Grzegorz Martens
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
aaa3d815c1
commit
1ad951e25b
@@ -26,6 +26,7 @@ let d = doc.dict(&obj); // Option<Dict>, resolving
|
||||
let data = stream.decoded()?; // filters applied on demand
|
||||
let data = doc.with_direct_parms(&stream).decoded()?; // /DecodeParms given by reference too
|
||||
let px = doc.decoded_image(&stream)?; // an image's samples, a CCITT fax image's too
|
||||
let fax = doc.fax_params(&stream); // a CCITT fax image's /DecodeParms, Table 11 defaults
|
||||
|
||||
doc.set(r, Object::Dict(d)); // edit: lands in the overlay
|
||||
let r = doc.add(obj); // append a new object
|
||||
|
||||
@@ -224,26 +224,15 @@ impl Document {
|
||||
self.resolve(o).as_dict().cloned()
|
||||
}
|
||||
|
||||
/// The samples of image `s`, decoded: through its filters ([`Stream::decoded`]), or, for a
|
||||
/// CCITT fax image (its last filter), through the filters before it and then
|
||||
/// [`pdfcraft_filters::decode_ccitt`] with its `/DecodeParms` (resolved; the dictionary of a
|
||||
/// lone filter, or its entry in an array), as `CCITTFaxDecode` gives them (ISO 32000-2 §7.4.6).
|
||||
/// Parameters given by reference count, for every filter ([`Document::with_direct_parms`]).
|
||||
/// A fax image's `/Columns` has to be its `/Width`. Up to its `/Height` rows are decoded,
|
||||
/// whatever `/Rows` says, within [`MAX_DECODED`] bytes; damaged data gives fewer, so a caller
|
||||
/// checks it has the rows it needs.
|
||||
pub fn decoded_image(&self, s: &Stream) -> Result<Vec<u8>, CosError> {
|
||||
self.decoded_image_within(s, MAX_DECODED)
|
||||
}
|
||||
|
||||
/// [`Document::decoded_image`] for an image whose decoding has a tighter bound than
|
||||
/// [`MAX_DECODED`]: no stage, the filters before a fax decoder included, may produce more than
|
||||
/// `max` bytes.
|
||||
pub fn decoded_image_within(&self, s: &Stream, max: usize) -> Result<Vec<u8>, CosError> {
|
||||
let max = max.min(MAX_DECODED);
|
||||
let s = &*self.with_direct_parms(s);
|
||||
let chain = s.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?;
|
||||
let Some(((pdfcraft_filters::Filter::CcittFax, _), before)) = chain.split_last() else { return s.decoded_within(max) };
|
||||
/// The CCITT parameters of image `s` when its last filter is `CCITTFaxDecode`: its
|
||||
/// `/DecodeParms` (resolved; the dictionary of a lone filter, or its entry in an array), with the
|
||||
/// defaults of ISO 32000-2 Table 11 for what that leaves out. `None` for another image, and for
|
||||
/// one whose `/Filter` isn't a name or an array of names.
|
||||
pub fn fax_params(&self, s: &Stream) -> Option<pdfcraft_filters::CcittParams> {
|
||||
let chain = s.readable_filters().ok()?;
|
||||
if !matches!(chain.last(), Some((pdfcraft_filters::Filter::CcittFax, _))) {
|
||||
return None;
|
||||
}
|
||||
// Its place in /Filter, /Crypt included, pairs it with its /DecodeParms entry (§7.4.1): the
|
||||
// last filter but a /Crypt, which `filters` leaves out wherever it stands.
|
||||
let index = match s.dict.get(b"Filter") {
|
||||
@@ -262,17 +251,38 @@ impl Document {
|
||||
_ => default,
|
||||
};
|
||||
let d = pdfcraft_filters::CcittParams::default();
|
||||
let p = pdfcraft_filters::CcittParams {
|
||||
Some(pdfcraft_filters::CcittParams {
|
||||
k: int(b"K", d.k),
|
||||
columns: int(b"Columns", d.columns),
|
||||
// As many rows as the image has: a /Rows below /Height would cut the picture short,
|
||||
// and rows past /Height aren't drawn.
|
||||
rows: 0,
|
||||
rows: int(b"Rows", d.rows),
|
||||
end_of_line: flag(b"EndOfLine", d.end_of_line),
|
||||
encoded_byte_align: flag(b"EncodedByteAlign", d.encoded_byte_align),
|
||||
end_of_block: flag(b"EndOfBlock", d.end_of_block),
|
||||
black_is_1: flag(b"BlackIs1", d.black_is_1),
|
||||
};
|
||||
})
|
||||
}
|
||||
|
||||
/// The samples of image `s`, decoded: through its filters ([`Stream::decoded`]), or, for a
|
||||
/// CCITT fax image (its last filter), through the filters before it and then
|
||||
/// [`pdfcraft_filters::decode_ccitt`] with its [`Document::fax_params`], as `CCITTFaxDecode`
|
||||
/// gives them (ISO 32000-2 §7.4.6). Parameters given by reference count, for every filter
|
||||
/// ([`Document::with_direct_parms`]). A fax image's `/Columns` has to be its `/Width`. Up to
|
||||
/// its `/Height` rows are decoded, whatever `/Rows` says, within [`MAX_DECODED`] bytes; damaged
|
||||
/// data gives fewer, so a caller checks it has the rows it needs.
|
||||
pub fn decoded_image(&self, s: &Stream) -> Result<Vec<u8>, CosError> {
|
||||
self.decoded_image_within(s, MAX_DECODED)
|
||||
}
|
||||
|
||||
/// [`Document::decoded_image`] for an image whose decoding has a tighter bound than
|
||||
/// [`MAX_DECODED`]: no stage, the filters before a fax decoder included, may produce more than
|
||||
/// `max` bytes.
|
||||
pub fn decoded_image_within(&self, s: &Stream, max: usize) -> Result<Vec<u8>, CosError> {
|
||||
let max = max.min(MAX_DECODED);
|
||||
let s = &*self.with_direct_parms(s);
|
||||
let Some(p) = self.fax_params(s) else { return s.decoded_within(max) };
|
||||
// As many rows as the image has: a /Rows below /Height would cut the picture short, and
|
||||
// rows past /Height aren't drawn.
|
||||
let p = pdfcraft_filters::CcittParams { rows: 0, ..p };
|
||||
if s.dict.int(b"Width") != Some(p.columns) {
|
||||
return Err(CosError::Filter("a fax image's /Columns aren't its /Width".into()));
|
||||
}
|
||||
@@ -280,9 +290,12 @@ impl Document {
|
||||
let stride = usize::try_from(p.columns).unwrap_or(usize::MAX).div_ceil(8).max(1);
|
||||
let rows = u32::try_from(max / stride).unwrap_or(u32::MAX).min(height);
|
||||
// The filters before it, such as ASCII85Decode, are undone first.
|
||||
let data = match before {
|
||||
[] => Cow::Borrowed(&s.raw[..]),
|
||||
_ => Cow::Owned(pdfcraft_filters::decode_tolerant(before, &s.raw, max).map_err(|e| CosError::Filter(e.to_string()))?.0),
|
||||
let chain = s.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?;
|
||||
let data = match chain.split_last() {
|
||||
Some((_, before)) if !before.is_empty() => {
|
||||
Cow::Owned(pdfcraft_filters::decode_tolerant(before, &s.raw, max).map_err(|e| CosError::Filter(e.to_string()))?.0)
|
||||
}
|
||||
_ => Cow::Borrowed(&s.raw[..]),
|
||||
};
|
||||
pdfcraft_filters::decode_ccitt(&data, &p, rows).map_err(|e| CosError::Filter(e.to_string()))
|
||||
}
|
||||
|
||||
@@ -131,7 +131,15 @@ fn fax_images_decode_with_their_parameters() {
|
||||
assert_eq!(image(&doc, Some(Object::Dict(by_reference)), 10, 3).unwrap(), bits, "/K and /Columns by reference");
|
||||
let mut understated = parms.clone();
|
||||
understated.set(b"Rows".to_vec(), Object::Int(1));
|
||||
assert_eq!(image(&doc, Some(Object::Dict(understated)), 10, 3).unwrap(), bits, "/Height rows, whatever /Rows says");
|
||||
assert_eq!(image(&doc, Some(Object::Dict(understated.clone())), 10, 3).unwrap(), bits, "/Height rows, whatever /Rows says");
|
||||
// The parameters as the file gives them, /Rows included, and Table 11's defaults.
|
||||
let mut d = Dict::new();
|
||||
d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode"));
|
||||
d.set(b"DecodeParms".to_vec(), Object::Dict(understated));
|
||||
let p = doc.fax_params(&Stream::from_raw(d.clone(), Vec::new())).unwrap();
|
||||
assert_eq!((p.k, p.columns, p.rows, p.end_of_block, p.black_is_1), (-1, 10, 1, true, false));
|
||||
d.set(b"Filter".to_vec(), Object::name("FlateDecode"));
|
||||
assert!(doc.fax_params(&Stream::from_raw(d, Vec::new())).is_none(), "not a fax image");
|
||||
assert!(matches!(image(&doc, Some(Object::Dict(parms.clone())), 12, 3), Err(CosError::Filter(_))), "/Columns other than /Width");
|
||||
assert!(matches!(image(&doc, None, 10, 3), Err(CosError::Filter(_))), "the default 1728 /Columns");
|
||||
let mut plain = Dict::new();
|
||||
|
||||
@@ -20,10 +20,10 @@ clear_marks(&mut doc, None)?; // remove marks without app
|
||||
- **images**: fully covered → removed; partly covered → a copy with the covered pixels cleared
|
||||
(8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax as the last
|
||||
filter, decoded through `Document::decoded_image_within`; `/DecodeParms` given by reference,
|
||||
numbers inside included, count; byte-aligned Group 3 with end-of-line codes but none before the
|
||||
first row, not as T.4 sends them, is cleared as PdfCraft draws it, which at `/K 0` may be other
|
||||
rows than were coded). Codecs redaction doesn't decode (DCT, JPX, JBIG2) are removed
|
||||
whole (fail-closed); inline images under an area are removed. Images with separate `/SMask`
|
||||
numbers inside included, count). Codecs redaction doesn't decode (DCT, JPX, JBIG2) are removed
|
||||
whole (fail-closed), and so is Group 3 on bytes (`/K` 0 or more, `/EncodedByteAlign`): PdfCraft's
|
||||
renderer draws some of its layouts (the fill ending each end-of-line code on a byte) as other
|
||||
rows than the decoder reads, so a mark over what is shown needn't cover what is cleared; inline images under an area are removed. Images with separate `/SMask`
|
||||
or `/Mask` data, and images whose data decodes to more than three times their declared raster
|
||||
and a byte per row, plus 4 KiB (ASCIIHex text broken into lines under Flate is a little over
|
||||
twice; for a fax image's code, about a byte per pixel), are removed whole;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! Clearing the pixels of an image under redaction regions. The image is decoded, every pixel
|
||||
//! whose footprint intersects a region is set to zero (no ink for image masks), and the result is
|
||||
//! written as a new Flate image; the original object is untouched (other pages may use it).
|
||||
//! CCITT fax images decode through `Document::decoded_image_within`. Images whose codec PdfCraft can't
|
||||
//! decode (DCT, JPX, JBIG2) return `None`, and the caller removes the whole image instead; that
|
||||
//! is fail-closed.
|
||||
//! CCITT fax images decode through `Document::decoded_image_within`, but for Group 3 on bytes,
|
||||
//! which PdfCraft's renderer may draw as other rows. Images whose codec PdfCraft can't decode
|
||||
//! (DCT, JPX, JBIG2), and Group 3 on bytes, return `None`, and the caller removes the whole image
|
||||
//! instead; that is fail-closed.
|
||||
|
||||
use pdfcraft_content::{Matrix, overlaps};
|
||||
use pdfcraft_cos::{Document, Object, Stream};
|
||||
@@ -68,6 +69,13 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]]
|
||||
if !ok {
|
||||
return None;
|
||||
}
|
||||
// Group 3 on bytes is removed whole for now. The decoder reads it by its end-of-line codes
|
||||
// when the data starts with one, but PdfCraft's renderer draws such rows (the fill ending
|
||||
// each code on a byte) as other rows, so the pixels a mark covers on screen needn't be the
|
||||
// ones cleared.
|
||||
if is_fax && doc.fax_params(s).is_none_or(|p| p.k >= 0 && p.encoded_byte_align) {
|
||||
return None;
|
||||
}
|
||||
let (w, h) = (s.dict.int(b"Width")?, s.dict.int(b"Height")?);
|
||||
if w <= 0 || h <= 0 || (w as u64) * (h as u64) > MAX_PIXELS {
|
||||
return None;
|
||||
|
||||
@@ -684,6 +684,40 @@ fn a_fax_image_that_cannot_be_decoded_in_full_is_removed_whole() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Group 3 on bytes is removed whole: the renderer draws some of its layouts as other rows than
|
||||
/// the decoder reads, so the pixels a mark covers on screen needn't be the ones cleared. Group 3
|
||||
/// off bytes and Group 4 on bytes are cleared. Each picture is 16 × 2 white, each row a white run
|
||||
/// of 16 (`101010`): off bytes; on bytes with no end-of-line codes; on bytes after codes whose fill
|
||||
/// ends each on a byte, at `/K 0` and at `/K 1` (a 1 after each code), `/EncodedByteAlign` given by
|
||||
/// reference there; and Group 4 rows on bytes, each the one code 1 (vertical, no change).
|
||||
#[test]
|
||||
fn group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared() {
|
||||
let image = "/Type /XObject /Subtype /Image /Width 16 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 1 /Filter /CCITTFaxDecode";
|
||||
for (k, on_bytes, parms, data, extra, removed) in [
|
||||
(0, false, "<< /K 0 /Columns 16 >>", vec![0xAA, 0xA0], vec![], false),
|
||||
(0, true, "<< /K 0 /Columns 16 /EncodedByteAlign true >>", vec![0xA8, 0xA8], vec![], true),
|
||||
(0, true, "<< /K 0 /Columns 16 /EncodedByteAlign true >>", vec![0x00, 0x01, 0xA8, 0x00, 0x01, 0xA8], vec![], true),
|
||||
(1, true, "<< /K 1 /Columns 16 /EncodedByteAlign 8 0 R >>", vec![0x00, 0x01, 0xD4, 0x00, 0x01, 0xD4], vec![b"true".to_vec()], true),
|
||||
(-1, true, "<< /K -1 /Columns 16 /EncodedByteAlign true >>", vec![0x80, 0x80], vec![], false),
|
||||
] {
|
||||
let p = pdfcraft_filters::CcittParams { k, columns: 16, encoded_byte_align: on_bytes, ..Default::default() };
|
||||
assert_eq!(pdfcraft_filters::decode_ccitt(&data, &p, 2).unwrap(), [0xFF; 4], "{parms}: the picture is white");
|
||||
let dict = format!("{image} /DecodeParms {parms}");
|
||||
let mut objs = vec![stream(&dict, data)];
|
||||
objs.extend(extra);
|
||||
let mut doc = one_page(b"q 160 0 0 20 0 0 cm /Im Do Q", "/XObject << /Im 7 0 R >>", objs);
|
||||
mark(&mut doc, 0, &[[0.0, 12.0, 75.0, 18.0]], "");
|
||||
let report = apply(&mut doc, None).unwrap();
|
||||
if removed {
|
||||
assert_eq!((report.images_cleared, report.images_removed), (0, 1), "{parms}");
|
||||
} else {
|
||||
assert_eq!((report.images_cleared, report.images_removed), (1, 0), "{parms}");
|
||||
let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
|
||||
assert_eq!(named_image(&saved, 0, b"PCRedacted1").as_deref(), Some(&[0x00, 0xFF, 0xFF, 0xFF][..]), "{parms}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A busy fax image is cleared, though its code is bigger than three times its raster (the bound
|
||||
/// for other images): a code takes up to about a byte per pixel, and the fax image gets that much.
|
||||
#[test]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# PDF specification and standards parity
|
||||
|
||||
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (§7.4: redaction decodes CCITT fax images; earlier, §7.4: CCITT decoding in `pdfcraft-filters`) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS)
|
||||
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (§7.4: redaction decodes CCITT fax images, Group 3 on bytes aside; earlier, §7.4: CCITT decoding in `pdfcraft-filters`) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS)
|
||||
|
||||
How much of ISO 32000-2 (PDF 2.0) and the PDF subset standards PdfCraft implements, set against
|
||||
what Acrobat Pro implements. This is PdfCraft's equivalent of a "geometry" or "codec" checklist:
|
||||
@@ -130,7 +130,7 @@ are borrowed; PDF 2.0 additions and most subset standards are missing. **150–2
|
||||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-26 | minor | §7.4: redaction clears pixels in CCITT fax images through the CCITT decoder, instead of removing them whole |
|
||||
| 2026-10-26 | minor | §7.4: redaction clears pixels in CCITT fax images through the CCITT decoder, instead of removing them whole (Group 4 and Group 3 off bytes; Group 3 on bytes is still removed whole, as the bootstrap renderer may draw it as other rows than the decoder reads) |
|
||||
| 2026-10-26 | minor | §7.4: CCITT decoding (`hayro-ccitt`) in `pdfcraft-filters`, for export and the Optimizer |
|
||||
| 2026-10-26 | minor | §7.4: CCITT Group 4 encoding (`fax`) in the Optimizer |
|
||||
| 2026-10-14 | minor | §12.4.4: `/Trans` and `/Dur` read into `DocInfo::transitions` and reported by `doc_info` |
|
||||
|
||||
@@ -131,7 +131,7 @@ status = "partial"
|
||||
commands = []
|
||||
tools = []
|
||||
evidence = ["crates/filters/src/ccitt.rs::decoding_gives_the_rows_back", "crates/filters/src/ccitt.rs::group_3_rows_on_bytes_read_by_the_codes_in_the_data", "crates/cos/src/document/tests.rs::fax_images_decode_with_their_parameters"]
|
||||
notes = "decoded through the hayro bootstrap for display; own codecs are M1.1/M1.2. CCITT fax decoding is in filters (decode_ccitt, through hayro-ccitt), which Document::decoded_image uses for Export ▸ Images, the optimizer and redaction. Byte-aligned Group 3 is read by its end-of-line codes when the data starts with one and on bytes otherwise; still limited: data with codes but none before the first row (not as T.4 sends them) is read on bytes, as the renderer draws it, and at /K 0 may come out as other rows, which the optimizer and redaction then write back"
|
||||
notes = "decoded through the hayro bootstrap for display; own codecs are M1.1/M1.2. CCITT fax decoding is in filters (decode_ccitt, through hayro-ccitt), which Document::decoded_image uses for Export ▸ Images, the optimizer and redaction. Byte-aligned Group 3 is read by its end-of-line codes when the data starts with one and on bytes otherwise; still limited: data with codes but none before the first row (not as T.4 sends them) is read on bytes, as the renderer draws it, and at /K 0 may come out as other rows, which the optimizer then writes back. The bootstrap renderer draws byte-aligned Group 3 whose fill ends each end-of-line code on a byte as other rows (black box, at every /K), which decode_ccitt reads right; redaction removes Group 3 on bytes whole"
|
||||
|
||||
[[feature]]
|
||||
id = "core.decompression-limits"
|
||||
@@ -5792,8 +5792,8 @@ milestone = "M8"
|
||||
status = "shipped"
|
||||
commands = ["redact.apply"]
|
||||
tools = ["redact_apply"]
|
||||
evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared"]
|
||||
notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax images (scans stored as Group 4) included; DCT/JPX/JBIG2 images removed whole"
|
||||
evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared"]
|
||||
notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax images (scans stored as Group 4) included; DCT/JPX/JBIG2 images, and Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), removed whole"
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-vectors"
|
||||
|
||||
Reference in New Issue
Block a user