M8: redaction stores again without them the images that share a redacted JBIG2 image's globals, so the globals leave the file (protect.redact-image-pixels)

A JBIG2 image's /JBIG2Globals hold the symbol dictionary of every image that
shares them (jbig2enc -s and most scanners give one to all of a scan's pages),
so the bitmaps of marks drawn only under a redaction stayed in the file while
another page's image used them. Every image using the globals of a removed or
replaced image (or of one in a removed or replaced form) is now decoded and
stored again in place as Flate without them; a full save then drops them. One
that can't be decoded for certain refuses the redaction
(RedactError::SharedJbig2Globals). Report::images_restored counts them.

crates/cos/src/document/objects.rs

@@ -326,6 +326,17 @@ impl Document {
         pdfcraft_filters::decode_ccitt(&data, &p, rows).map_err(filter)
     }
 
+    /// The `/JBIG2Globals` stream a JBIG2 image `s` names by reference in its last filter's
+    /// `/DecodeParms`: `None` for a stream whose last filter isn't `JBIG2Decode`, or one without
+    /// globals. Redaction finds with it the other images that share a redacted image's globals.
+    pub fn jbig2_globals_ref(&self, s: &Stream) -> Option<ObjRef> {
+        let chain = s.readable_filters().ok()?;
+        if !matches!(chain.last(), Some((pdfcraft_filters::Filter::Jbig2, _))) {
+            return None;
+        }
+        self.last_filter_parms(s).get(b"JBIG2Globals").and_then(Object::as_ref)
+    }
+
     /// What the `/JBIG2Globals` stream of JBIG2 image `s` (in its last filter's `/DecodeParms`)
     /// decodes to, within [`MAX_JBIG2_GLOBALS`]; `None` without one (a reference to no object is
     /// one to null, ISO 32000-2 §7.3.10, which is an absent entry, §7.3.7). The image is then read

crates/redact/README.md

@@ -21,9 +21,11 @@ clear_marks(&mut doc, None)?;                        // remove marks without app
   (8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax or JBIG2 as
   the last filter, decoded through `Document::decoded_image_within`, a JBIG2 image with its
   `/JBIG2Globals`; `/DecodeParms` given by reference, numbers inside included, count). The copy
-  is Flate, and refers to no globals: when no other image uses them, they leave the file with the
-  original on the full rewrite; when one does, they stay as they are, and may hold the shapes of
-  marks under the area (not where they were). Codecs redaction doesn't decode (DCT, JPX) are
+  is Flate, and refers to no globals. Globals hold the shapes of every image that shares them (a
+  scan's pages, often), those of marks under the area included, so every other image that uses
+  the globals of a removed or replaced image is decoded and stored again in place as Flate without
+  them (`Report::images_restored`), and they leave the file on the full rewrite; one that can't be
+  decoded for certain refuses the redaction (`RedactError::SharedJbig2Globals`). Codecs redaction doesn't decode (DCT, JPX) are
   removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer
   draws the whole page into the image, so the rows cleared wouldn't be the ones shown) or of more
   than one bit or component, and Group 3 on bytes (`/K` 0 or more, `/EncodedByteAlign`): PdfCraft's

crates/redact/src/image.rs

@@ -167,3 +167,34 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]]
     dict.remove(b"Alternates");
     Some(Some(Stream::flate(dict, &data)))
 }
+
+/// JBIG2 image `s` stored again without its `/JBIG2Globals`: decoded through them, as it is drawn,
+/// and written as a Flate image of the same samples, its other entries kept. `None` when it can't
+/// be decoded for certain (too large, data that doesn't decode, a JBIG2 page of another size than
+/// the image, or not a one-bit image of one component). Redaction does this to every image that
+/// shares a redacted image's globals, since their symbol dictionary holds the shapes of marks drawn
+/// under the area too.
+pub(crate) fn without_globals(doc: &Document, s: &Stream) -> Option<Stream> {
+    let (w, h) = (s.dict.int(b"Width")?, s.dict.int(b"Height")?);
+    if w <= 0 || h <= 0 || (w as u64) * (h as u64) > MAX_PIXELS {
+        return None;
+    }
+    let (w, h) = (w as usize, h as usize);
+    let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true)));
+    // JBIG2 codes one-bit images of one component (ISO 32000-2 §7.4.7).
+    if !mask && (components(doc, s)?, s.dict.int(b"BitsPerComponent")?) != (1, 1) {
+        return None;
+    }
+    let bytes = w.div_ceil(8).checked_mul(h)?;
+    // As for `clear`: the filters before the JBIG2 decoder may give up to three bytes for each one
+    // of the raster and a byte a row.
+    let allowance = bytes.checked_add(h)?.checked_mul(3)?.checked_add(4096)?;
+    let mut data = doc.decoded_image_within(s, allowance).ok()?;
+    if data.len() < bytes {
+        return None;
+    }
+    data.truncate(bytes);
+    let mut dict = s.dict.clone();
+    dict.remove(b"Length");
+    Some(Stream::flate(dict, &data))
+}

crates/redact/src/lib.rs

@@ -70,6 +70,12 @@ pub enum RedactError {
     /// could not be shown complete.
     #[error("the document's objects nest too deeply to remove hidden information completely; the operation was refused")]
     TooDeep,
+    /// A JBIG2 image shares its `/JBIG2Globals` with a redacted one and can't be stored again
+    /// without them, so the shapes they hold, those of marks under the area included, would stay.
+    #[error(
+        "{0} JBIG2 image(s) share their symbol data with a redacted image and can't be decoded to be stored without it, so that data, which can show what is under the marks, would stay in the file; nothing was redacted"
+    )]
+    SharedJbig2Globals(usize),
     #[error(transparent)]
     Edit(#[from] pdfcraft_edit::EditError),
     #[error(transparent)]
@@ -132,6 +138,9 @@ pub struct Report {
     pub glyphs: usize,
     pub images_removed: usize,
     pub images_cleared: usize,
+    /// JBIG2 images, anywhere in the document, stored again as Flate without the `/JBIG2Globals`
+    /// they shared with a redacted image, so those globals leave the file.
+    pub images_restored: usize,
     pub paths_removed: usize,
     pub paths_clipped: usize,
     pub forms_rewritten: usize,
@@ -509,6 +518,7 @@ fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Redac
     // A removed image or form must not stay reachable through another resource dictionary (an
     // inherited or shared /Resources): a full save keeps whatever is still referenced.
     interp::unlink_doomed(doc, &doomed_xobjects, &retired_properties)?;
+    report.images_restored = restore_shared_jbig2(doc, &doomed_xobjects)?;
 
     // 5. Verify: nothing readable may remain under a region.
     let mut residue = 0;
@@ -533,6 +543,69 @@ fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Redac
     Ok(report)
 }
 
+/// Most XObjects followed down from the removed and replaced ones to find their JBIG2 images.
+const MAX_JBIG2_WALK: usize = 100_000;
+
+/// The `/JBIG2Globals` of a redacted JBIG2 image hold the symbol dictionary of every image that
+/// shares them (`jbig2enc -s` and most scanners give one to all the pages of a scan): the bitmaps
+/// of marks drawn only under the area too. The image itself is replaced or removed, but the globals
+/// would stay for as long as another image uses them. So every image that uses the globals of a
+/// removed or replaced image (`doomed`), or of one drawn in a removed or replaced form, is decoded
+/// and stored again in place as Flate, without them, and a full save then leaves them out. An
+/// image that can't be stored again refuses the redaction (nothing changes, as `apply` works on a
+/// copy), but for a removed or replaced one, which is either unlinked or still drawn whole on
+/// another page. Returns how many images that weren't removed or replaced were stored again.
+fn restore_shared_jbig2(doc: &mut Document, doomed: &[ObjRef]) -> Result<usize, RedactError> {
+    let mut globals = HashSet::new();
+    let mut seen = HashSet::new();
+    let mut stack: Vec<ObjRef> = doomed.to_vec();
+    while let Some(r) = stack.pop() {
+        if !seen.insert(r) || seen.len() > MAX_JBIG2_WALK {
+            continue;
+        }
+        let object = doc.get(r);
+        let Object::Stream(s) = &*object else { continue };
+        if let Some(g) = doc.jbig2_globals_ref(s) {
+            globals.insert(g);
+        }
+        // A form's images, and its forms' (an image's /SMask is a stream of its own, found below).
+        let resources = s.dict.get(b"Resources").map(|r| doc.resolve(r));
+        let xobjects = resources.as_deref().and_then(Object::as_dict).and_then(|d| d.get(b"XObject")).map(|x| doc.resolve(x));
+        if let Some(x) = xobjects.as_deref().and_then(Object::as_dict) {
+            stack.extend(x.iter().filter_map(|(_, v)| v.as_ref()));
+        }
+        if let Some(m) = s.dict.get(b"SMask").and_then(Object::as_ref) {
+            stack.push(m);
+        }
+    }
+    if globals.is_empty() {
+        return Ok(0);
+    }
+    let sharing: Vec<ObjRef> = doc
+        .scan_objects()
+        .filter(|(_, o)| matches!(&**o, Object::Stream(s) if doc.jbig2_globals_ref(s).is_some_and(|g| globals.contains(&g))))
+        .map(|(r, _)| r)
+        .collect();
+    let doomed: HashSet<ObjRef> = doomed.iter().copied().collect();
+    let (mut restored, mut stuck) = (0, 0);
+    for r in sharing {
+        let object = doc.get(r);
+        let Object::Stream(s) = &*object else { continue };
+        match image::without_globals(doc, s) {
+            Some(new) => {
+                doc.set(r, Object::Stream(new));
+                restored += usize::from(!doomed.contains(&r));
+            }
+            None if doomed.contains(&r) => {}
+            None => stuck += 1,
+        }
+    }
+    if stuck > 0 {
+        return Err(RedactError::SharedJbig2Globals(stuck));
+    }
+    Ok(restored)
+}
+
 /// What applying the marks collects across pages.
 struct Applying {
     report: Report,

crates/redact/src/tests.rs

@@ -938,6 +938,133 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {
     }
 }
 
+/// The segments of a JBIG2 stream `encode_jbig2` wrote, each with its header: a header is the
+/// segment's number (4 bytes), type, flags whose top three bits count the segments it refers to,
+/// those (a byte each), its page (a byte) and its data's length (4 bytes; ISO/IEC 14492 7.2).
+fn jbig2_segments(mut coded: &[u8]) -> Vec<Vec<u8>> {
+    let mut segments = Vec::new();
+    while !coded.is_empty() {
+        let refers = usize::from(coded[5] >> 5);
+        let header = 4 + 1 + 1 + refers + 1 + 4;
+        let length = u32::from_be_bytes(coded[header - 4..header].try_into().unwrap()) as usize;
+        segments.push(coded[..header + length].to_vec());
+        coded = &coded[header + length..];
+    }
+    segments
+}
+
+/// Two pages of a scan share one `/JBIG2Globals` stream holding the symbol dictionary, as
+/// `jbig2enc -s` and most scanners write them, and the first has a mark found nowhere else (a
+/// signature, say) under the area. The dictionary holds that mark's bitmap, so once the first page
+/// is redacted the second page's image is stored again without the globals, and neither the
+/// dictionary nor any stream naming globals stays in the saved file; both pages keep every pixel
+/// outside the area.
+#[test]
+fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() {
+    const LETTER: [&str; 7] = ["#####", "#....", "####.", "#....", "#....", "#....", "#####"];
+    const SIGNATURE: [&str; 7] = ["#..#.##", ".##.#..", "#.#..##", "..##.#.", "##..#.#", ".#.##..", "#..#..#"];
+    let (w, h) = (400usize, 40usize);
+    let stride = w.div_ceil(8);
+    let draw = |bits: &mut [u8], shape: &[&str], top: usize, left: usize| {
+        for (dy, row) in shape.iter().enumerate() {
+            for (dx, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'#') {
+                bits[(top + dy) * stride + (left + dx) / 8] &= !(0x80 >> ((left + dx) % 8));
+            }
+        }
+    };
+    // The letter over and over lower down, and the signature once, top left, under the mark.
+    let mut first = vec![0xFFu8; stride * h];
+    for top in [14usize, 24, 32] {
+        for x in (3..w - 6).step_by(7) {
+            draw(&mut first, &LETTER, top, x);
+        }
+    }
+    draw(&mut first, &SIGNATURE, 2, 10);
+    let mut second = vec![0xFFu8; stride * h];
+    for x in (3..w - 6).step_by(7) {
+        draw(&mut second, &LETTER, 20, x);
+    }
+    // The first page's symbol dictionary (its second segment) goes to the globals, on no page; the
+    // second page's segments are numbered past it.
+    let mut segments = jbig2_segments(&pdfcraft_filters::encode_jbig2(&first, w as u32, h as u32).unwrap());
+    assert_eq!(segments[1][4], 0, "the second segment is the symbol dictionary");
+    let mut dictionary = segments.remove(1);
+    dictionary[6] = 0;
+    let page_one: Vec<u8> = segments.concat();
+    let page_two: Vec<u8> = jbig2_segments(&pdfcraft_filters::encode_jbig2_generic(&second, w as u32, h as u32).unwrap())
+        .into_iter()
+        .enumerate()
+        .flat_map(|(k, mut s)| {
+            s[..4].copy_from_slice(&(100 + k as u32).to_be_bytes());
+            s
+        })
+        .collect();
+    let image = format!(
+        "/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode /DecodeParms << /JBIG2Globals 9 0 R >>"
+    );
+    // Half a point a pixel, the image's top row at y 120.
+    let page = |content: u32, im: u32| {
+        format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents {content} 0 R /Resources << /XObject << /Im {im} 0 R >> >> >>")
+            .into_bytes()
+    };
+    let build = |second_image: &str| {
+        pdf(vec![
+            b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
+            b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(),
+            page(5, 7),
+            page(6, 8),
+            stream("", b"q 200 0 0 20 0 100 cm /Im Do Q".as_slice()),
+            stream("", b"q 200 0 0 20 0 100 cm /Im Do Q".as_slice()),
+            stream(&image, page_one.clone()),
+            stream(second_image, page_two.clone()),
+            stream("", dictionary.clone()),
+        ])
+    };
+    let mut doc = build(&image);
+    // A page's picture as it is drawn, JBIG2 read with its globals.
+    let picture = |doc: &Document, page: usize, name: &[u8]| {
+        let p = pdfcraft_model::pages(doc).swap_remove(page);
+        let res = doc.resolve(p.dict.get(b"Resources")?).as_dict().cloned()?;
+        let xo = doc.resolve(res.get(b"XObject")?).as_dict().cloned()?;
+        match &*doc.get(xo.get(name)?.as_ref()?) {
+            Object::Stream(s) => doc.decoded_image(s).ok(),
+            _ => None,
+        }
+    };
+    assert_eq!(picture(&doc, 0, b"Im"), Some(first.clone()), "the first page reads its symbols from the globals");
+    assert_eq!(picture(&doc, 1, b"Im"), Some(second.clone()));
+    // Columns 0 to 99 of rows 0 to 12.
+    mark(&mut doc, 0, &[[0.25, 113.75, 49.75, 120.0]], "");
+    let report = apply(&mut doc, None).unwrap();
+    assert_eq!((report.images_cleared, report.images_removed, report.images_restored), (1, 0, 1));
+    let mut cleared = first.clone();
+    for y in 0..=12 {
+        for x in 0..=99 {
+            cleared[y * stride + x / 8] &= !(0x80 >> (x % 8));
+        }
+    }
+    let saved = reopen(&doc);
+    assert_eq!(picture(&saved, 0, b"PCRedacted1"), Some(cleared));
+    assert_eq!(picture(&saved, 1, b"Im"), Some(second), "the second page keeps its picture");
+    // The dictionary's data (after its 11-byte header) codes the signature's bitmap.
+    let coded = &dictionary[11..];
+    for (r, object) in saved.scan_objects() {
+        let mut bytes = Vec::new();
+        pdfcraft_cos::serialize(&object, &mut bytes);
+        assert!(!bytes.windows(b"JBIG2Globals".len()).any(|w| w == b"JBIG2Globals"), "{r:?} still names globals");
+        let data = match &*object {
+            Object::Stream(s) => s.decoded().unwrap_or_default(),
+            _ => Vec::new(),
+        };
+        assert!(!bytes.windows(coded.len()).chain(data.windows(coded.len())).any(|w| w == coded), "the symbol dictionary stays as {r:?}");
+    }
+    // The second page's image can't be read for certain (its JBIG2 page is shorter than the
+    // image), so it can't be stored without the globals: the redaction is refused.
+    let mut doc = build(&image.replace(&format!("/Height {h}"), &format!("/Height {}", h + 1)));
+    mark(&mut doc, 0, &[[0.25, 113.75, 49.75, 120.0]], "");
+    assert!(matches!(apply(&mut doc, None), Err(RedactError::SharedJbig2Globals(1))));
+}
+
 /// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through
 /// before (inherited from the /Pages node, or shared with another page) still named the removed
 /// image and form, so a full-rewrite save kept them.

parity/acrobat-features.toml

@@ -5804,8 +5804,8 @@ milestone = "M8"
 status = "shipped"
 commands = ["redact.apply"]
 tools = ["redact_apply"]
-evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals"]
-notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)"
+evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"]
+notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; every other image using a redacted image's globals is stored again as Flate without them, so they leave the file (one that can't be decoded refuses the redaction)"
 
 [[feature]]
 id = "protect.redact-vectors"