diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 8d1515d8a4..fe21d13b32 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,8 +17,9 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +# CCITT Group 4 and JBIG2 encoding for cleared one-bit images. +pdfcraft-filters = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -pdfcraft-filters = { workspace = true } pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index af43e958d8..e52c1d8b98 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -21,7 +21,8 @@ clear_marks(&mut doc, None)?; // remove marks without app (8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax or JBIG2 as the last filter, decoded through `Document::decoded_image_within`, a JBIG2 image with its `/JBIG2Globals`; `/DecodeParms` given by reference, numbers inside included, count). The copy - is Flate, and refers to no globals: when no other image uses them, they leave the file with the + is Flate, or for a one-bit image of one component (a scan as a rule) CCITT Group 4, or JBIG2 + when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to no globals: when no other image uses them, they leave the file with the original on the full rewrite; when one does, they stay as they are, and may hold the shapes of marks under the area (not where they were). Codecs redaction doesn't decode (DCT, JPX) are removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer @@ -109,5 +110,4 @@ Patterns are heuristic: dates, protocol identifiers and longer number runs are r Custom code sets aren't supported yet. Not yet: pattern locales, struct-tree/alt-text cleanup, metadata and hidden-information -sanitising (M8.4), re-encoding of DCT images (they are removed instead), and storing a cleared black-and-white -image as CCITT Group 4 or JBIG2 again (it is Flate, larger; the optimizer makes it smaller). +sanitising (M8.4), re-encoding of DCT images (they are removed instead). diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 6c7a6bf93b..e9606c5e30 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -1,13 +1,14 @@ //! Clearing the pixels of an image under redaction regions. The image is decoded, every pixel //! whose footprint intersects a region is set to zero (no ink for image masks), and the result is -//! written as a new Flate image; the original object is untouched (other pages may use it). +//! written as a new image, Flate, or for a one-bit image of one component Group 4, or JBIG2 when it +//! was JBIG2, whichever is smallest; the original object is untouched (other pages may use it). //! CCITT fax and JBIG2 images decode through `Document::decoded_image_within`, but for Group 3 on //! bytes, which PdfCraft's renderer may draw as other rows, and a JBIG2 page of other rows than the //! image's. Images whose codec PdfCraft can't decode (DCT, JPX), and those, return `None`, and the //! caller removes the whole image instead; that is fail-closed. use pdfcraft_content::{Matrix, overlaps}; -use pdfcraft_cos::{Document, Object, Stream}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed). const MAX_PIXELS: u64 = 64 << 20; @@ -165,5 +166,39 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]] dict.remove(b"Length"); dict.remove(b"DecodeParms"); dict.remove(b"Alternates"); - Some(Some(Stream::flate(dict, &data))) + Some(Some(smallest(dict, &data, w, h, ncomp == 1 && bpc == 1, is_jbig2))) +} + +/// The cleared samples `data` of a `w` × `h` image as a stream with `dict`: Flate, or, for one bit +/// of one component (`bilevel`, a scan as a rule, which Flate stores several times larger), CCITT +/// Group 4, or JBIG2 when the image was JBIG2 (`jbig2`; JBIG2 is PDF 1.4, which such a file already +/// is), whichever is smallest as stored, dictionary included (a tiny image can lose by Group 4's +/// parameters). Both decode to `data`, cleared before it is coded, so nothing of the original's +/// code is carried over; Group 4 or JBIG2 that can't code the rows (too wide) is left out. +fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream { + let flate = Stream::flate(dict.clone(), data); + let Some((columns, rows)) = u32::try_from(w).ok().zip(u32::try_from(h).ok()).filter(|_| bilevel) else { return flate }; + let g4 = pdfcraft_filters::encode_g4(data, columns, rows).ok().map(|coded| { + let mut dict = dict.clone(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(columns))); + parms.set(b"Rows".to_vec(), Object::Int(i64::from(rows))); + dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + Stream::from_raw(dict, coded) + }); + let jbig = jbig2.then(|| pdfcraft_filters::encode_jbig2(data, columns, rows).ok()).flatten().map(|coded| { + let mut dict = dict.clone(); + dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + Stream::from_raw(dict, coded) + }); + [g4, jbig].into_iter().flatten().fold(flate, |best, s| if stored_len(&s) < stored_len(&best) { s } else { best }) +} + +/// The bytes stream `s` takes in a file: its data and its dictionary as written. +fn stored_len(s: &Stream) -> usize { + let mut dict = Vec::new(); + pdfcraft_cos::serialize(&Object::Dict(s.dict.clone()), &mut dict); + dict.len().saturating_add(s.raw.len()) } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 08faef69a7..2e76bb0bac 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -519,13 +519,19 @@ fn two_pages_one_resource_dictionary(inherited: bool, again: bool) -> Document { ]) } -/// The samples of the image a page's (possibly inherited) resources name `name`. +/// The samples of the image a page's (possibly inherited) resources name `name`, decoded through +/// its image codec too (`Document::decoded_image`). fn named_image(doc: &Document, page: usize, name: &[u8]) -> Option> { + named_stream(doc, page, name).and_then(|s| doc.decoded_image(&s).ok()) +} + +/// The image a page's (possibly inherited) resources name `name`. +fn named_stream(doc: &Document, page: usize, name: &[u8]) -> Option { let p = pdfcraft_model::pages(doc).swap_remove(page); let res = doc.resolve(p.dict.get(b"Resources")?).as_dict().cloned()?; let xo = doc.resolve(res.get(b"XObject")?).as_dict().cloned()?; match &*doc.get(xo.get(name)?.as_ref()?) { - Object::Stream(s) => s.decoded().ok(), + Object::Stream(s) => Some(s.clone()), _ => None, } } @@ -833,7 +839,8 @@ fn a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole() { /// A scan stored as JBIG2 symbols, with globals, keeps every pixel outside the area once it is /// redacted and saved, and the pixels under it are cleared; neither its JBIG2 code nor its globals, -/// which no other image uses, stay in the file. +/// which no other image uses, stay in the file. The cleared copy is coded afresh as JBIG2, smaller +/// than Flate, and refers to no globals. #[test] fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { const LETTERS: [[&str; 7]; 3] = [ @@ -854,6 +861,7 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { } } let coded = pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap(); + let original = coded.clone(); assert!(coded.len() < pdfcraft_filters::encode_jbig2_generic(&bits, w as u32, h as u32).unwrap().len(), "symbol coded"); let dict = format!( "/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode /DecodeParms << /JBIG2Globals 8 0 R >>" @@ -877,13 +885,111 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); assert_eq!(named_image(&saved, 0, b"PCRedacted1"), Some(cleared)); assert_no_secret(&doc, b"LEAK_GLOBALS"); + let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap(); + assert_eq!(copy.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..])); + assert!(!copy.dict.contains(b"DecodeParms"), "the copy refers to no globals"); + let flate = pdfcraft_filters::encode_flate(&named_image(&saved, 0, b"PCRedacted1").unwrap()); + assert!(copy.raw.len() < flate.len(), "JBIG2 {} bytes, Flate {}", copy.raw.len(), flate.len()); for (r, object) in saved.scan_objects() { if let Object::Stream(s) = &*object { - assert_ne!(s.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "the JBIG2 image stays as {r:?}"); + assert_ne!(&s.raw[..], &original[..], "the original JBIG2 code stays as {r:?}"); } } } +/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when +/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it +/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples. +#[test] +fn a_cleared_black_and_white_scan_is_stored_as_group_4() { + // 800 × 200: lines of word-sized dark blocks 14 rows tall, their edges ragged by a sample from + // row to row, as a scan's are; Group 4 codes such rows from the row above. + let (w, h) = (800usize, 200usize); + let stride = w / 8; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut bits = vec![0xFFu8; stride * h]; + for top in (3..h - 14).step_by(20) { + let mut x = 4; + while x + 70 < w { + let len = 20 + (next() % 40) as usize; + for y in top..top + 14 { + let (left, right) = (x + (next() % 2) as usize, x + len - (next() % 2) as usize); + for xx in left..right { + bits[y * stride + xx / 8] &= !(0x80 >> (xx % 8)); + } + } + x += len + 6 + (next() % 6) as usize; + } + } + let g4 = pdfcraft_filters::encode_g4(&bits, w as u32, h as u32).unwrap(); + assert!(g4.len() < pdfcraft_filters::encode_flate(&bits).len(), "Group 4 is smaller than Flate here"); + let grey: Vec = (0..w * h).map(|i| (i % 251) as u8).collect(); + for (dict, data, filter) in [ + ( + format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"), + pdfcraft_filters::encode_flate(&bits), + &b"CCITTFaxDecode"[..], + ), + ( + format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"), + pdfcraft_filters::encode_flate(&grey), + &b"FlateDecode"[..], + ), + ] { + let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]); + // The left quarter of the top half. + mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{dict}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap(); + assert_eq!(copy.dict.name(b"Filter"), Some(filter), "{dict}"); + // Covered: rows 0 to 99 (y 150 down to 125, a quarter point a row), samples 0 to 199. + let mut expected = if filter == b"FlateDecode" { grey.clone() } else { bits.clone() }; + for y in 0..100 { + for x in 0..200 { + if filter == b"FlateDecode" { + expected[y * w + x] = 0; + } else { + expected[y * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap(), expected, "{dict}"); + } +} + +/// A cleared one-bit image is stored as the smallest stream, its dictionary counted: a tiny one, +/// whose Group 4 code is shorter than Flate's but whose `/DecodeParms` aren't, stays Flate. Rows +/// wider than Group 4 codes (`MAX_G4_COLUMNS`) stay Flate too, and still clear. +#[test] +fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() { + let wide = pdfcraft_filters::MAX_G4_COLUMNS as usize + 1; + for (w, h) in [(3usize, 2usize), (wide, 2)] { + let stride = w.div_ceil(8); + let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"); + let mut doc = one_page( + b"q 200 0 0 20 0 100 cm /Im Do Q", + "/XObject << /Im 7 0 R >>", + vec![stream(&dict, pdfcraft_filters::encode_flate(&vec![0xFF; stride * h]))], + ); + // The top row's left half. + mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{w} × {h}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..]), "{w} × {h}"); + let samples = named_image(&saved, 0, b"PCRedacted1").unwrap(); + let sample = |x: usize| samples.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + assert!(!sample(0) && sample(w - 1), "{w} × {h}: the top row's first sample is cleared and its last kept"); + assert!(samples.get(stride..).is_some_and(|row| row.iter().all(|b| *b == 0xFF)), "{w} × {h}: the bottom row is kept"); + } +} + /// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through /// before (inherited from the /Pages node, or shared with another page) still named the removed /// image and form, so a full-rewrite save kept them. @@ -2525,7 +2631,8 @@ fn assert_no_secret(doc: &Document, secret: &[u8]) { pdfcraft_cos::serialize(&object, &mut bytes); assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}"); if let Object::Stream(s) = &*object { - let data = s.decoded().unwrap(); + // Image codecs, a cleared scan coded as Group 4 or JBIG2, decode through the document. + let data = s.decoded().or_else(|_| saved.decoded_image(s)).unwrap(); assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}"); } } diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 4878eac627..de1d9d9e38 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5804,8 +5804,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals"] -notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)" +evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4"] +notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)" [[feature]] id = "protect.redact-vectors"