diff --git a/Cargo.lock b/Cargo.lock index a950149243..6518430017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3936,7 +3936,7 @@ dependencies = [ [[package]] name = "pdfcraft" -version = "0.11.0" +version = "0.12.0" dependencies = [ "chrono", "craft-quit", @@ -3962,7 +3962,7 @@ dependencies = [ [[package]] name = "pdfcraft-a11y" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -3974,7 +3974,7 @@ dependencies = [ [[package]] name = "pdfcraft-annot" -version = "0.11.0" +version = "0.12.0" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -3985,7 +3985,7 @@ dependencies = [ [[package]] name = "pdfcraft-audit" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -3996,7 +3996,7 @@ dependencies = [ [[package]] name = "pdfcraft-automation" -version = "0.11.0" +version = "0.12.0" dependencies = [ "base64 0.22.1", "flate2", @@ -4018,7 +4018,7 @@ dependencies = [ [[package]] name = "pdfcraft-cli" -version = "0.11.0" +version = "0.12.0" dependencies = [ "base64 0.22.1", "pdfcraft-audit", @@ -4036,18 +4036,19 @@ dependencies = [ [[package]] name = "pdfcraft-compare" -version = "0.11.0" +version = "0.12.0" [[package]] name = "pdfcraft-content" -version = "0.11.0" +version = "0.12.0" dependencies = [ + "hayro-syntax", "pdfcraft-cos", ] [[package]] name = "pdfcraft-cos" -version = "0.11.0" +version = "0.12.0" dependencies = [ "chrono", "hayro-syntax", @@ -4062,7 +4063,7 @@ dependencies = [ [[package]] name = "pdfcraft-create" -version = "0.11.0" +version = "0.12.0" dependencies = [ "encoding_rs", "hayro-syntax", @@ -4078,7 +4079,7 @@ dependencies = [ [[package]] name = "pdfcraft-crypt" -version = "0.11.0" +version = "0.12.0" dependencies = [ "aes", "md-5", @@ -4090,7 +4091,7 @@ dependencies = [ [[package]] name = "pdfcraft-edit" -version = "0.11.0" +version = "0.12.0" dependencies = [ "hayro-syntax", "kurbo 0.13.1", @@ -4105,7 +4106,7 @@ dependencies = [ [[package]] name = "pdfcraft-engine" -version = "0.11.0" +version = "0.12.0" dependencies = [ "chrono", "craft-docpath", @@ -4146,14 +4147,14 @@ dependencies = [ [[package]] name = "pdfcraft-export" -version = "0.11.0" +version = "0.12.0" dependencies = [ "flate2", ] [[package]] name = "pdfcraft-filters" -version = "0.11.0" +version = "0.12.0" dependencies = [ "flate2", "proptest", @@ -4163,7 +4164,7 @@ dependencies = [ [[package]] name = "pdfcraft-fonts" -version = "0.11.0" +version = "0.12.0" dependencies = [ "harfrust", "pdfcraft-cos", @@ -4173,7 +4174,7 @@ dependencies = [ [[package]] name = "pdfcraft-forms" -version = "0.11.0" +version = "0.12.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4186,11 +4187,11 @@ dependencies = [ [[package]] name = "pdfcraft-geom" -version = "0.11.0" +version = "0.12.0" [[package]] name = "pdfcraft-js" -version = "0.11.0" +version = "0.12.0" dependencies = [ "boa_engine", "chrono", @@ -4198,7 +4199,7 @@ dependencies = [ [[package]] name = "pdfcraft-measure" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4211,14 +4212,14 @@ dependencies = [ [[package]] name = "pdfcraft-model" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-cos", ] [[package]] name = "pdfcraft-ocr" -version = "0.11.0" +version = "0.12.0" dependencies = [ "ocrs", "pdfcraft-fonts", @@ -4228,7 +4229,7 @@ dependencies = [ [[package]] name = "pdfcraft-optimize" -version = "0.11.0" +version = "0.12.0" dependencies = [ "image", "pdfcraft-annot", @@ -4241,11 +4242,11 @@ dependencies = [ [[package]] name = "pdfcraft-oracle" -version = "0.11.0" +version = "0.12.0" [[package]] name = "pdfcraft-organize" -version = "0.11.0" +version = "0.12.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4256,7 +4257,7 @@ dependencies = [ [[package]] name = "pdfcraft-platform" -version = "0.11.0" +version = "0.12.0" dependencies = [ "async-channel", "futures-lite", @@ -4266,7 +4267,7 @@ dependencies = [ [[package]] name = "pdfcraft-preflight" -version = "0.11.0" +version = "0.12.0" dependencies = [ "flate2", "pdfcraft-content", @@ -4277,7 +4278,7 @@ dependencies = [ [[package]] name = "pdfcraft-print" -version = "0.11.0" +version = "0.12.0" dependencies = [ "craft-print", "pdfcraft-content", @@ -4289,7 +4290,7 @@ dependencies = [ [[package]] name = "pdfcraft-redact" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4305,7 +4306,7 @@ dependencies = [ [[package]] name = "pdfcraft-render" -version = "0.11.0" +version = "0.12.0" dependencies = [ "bytemuck", "flate2", @@ -4324,7 +4325,7 @@ dependencies = [ [[package]] name = "pdfcraft-sign" -version = "0.11.0" +version = "0.12.0" dependencies = [ "aes", "aws-lc-rs", @@ -4359,11 +4360,11 @@ dependencies = [ [[package]] name = "pdfcraft-testkit" -version = "0.11.0" +version = "0.12.0" [[package]] name = "pdfcraft-ui-egui" -version = "0.11.0" +version = "0.12.0" dependencies = [ "arboard", "base64 0.22.1", @@ -4408,7 +4409,7 @@ dependencies = [ [[package]] name = "pdfcraft-web" -version = "0.11.0" +version = "0.12.0" dependencies = [ "eframe", "egui", @@ -4420,7 +4421,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfa" -version = "0.11.0" +version = "0.12.0" dependencies = [ "base64 0.22.1", "image", @@ -4434,7 +4435,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfdf" -version = "0.11.0" +version = "0.12.0" dependencies = [ "pdfcraft-annot", "pdfcraft-cos", diff --git a/Cargo.toml b/Cargo.toml index 2012403144..5841b50c68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor"] default-members = ["crates/*", "apps/pdfcraft", "apps/pdfcraft-cli", "xtask"] [workspace.package] -version = "0.11.0" +version = "0.12.0" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -21,39 +21,39 @@ craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1. # Whether a path stored in a document may be opened (GoToR link targets). craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "82a24b006eefa036b43c09a67fb9a3fbd17ffe63" } # internal -pdfcraft-geom = { path = "crates/geom", version = "0.11.0" } -pdfcraft-filters = { path = "crates/filters", version = "0.11.0" } -pdfcraft-cos = { path = "crates/cos", version = "0.11.0" } -pdfcraft-crypt = { path = "crates/crypt", version = "0.11.0" } -pdfcraft-organize = { path = "crates/organize", version = "0.11.0" } -pdfcraft-measure = { path = "crates/measure", version = "0.11.0" } -pdfcraft-annot = { path = "crates/annot", version = "0.11.0" } -pdfcraft-audit = { path = "crates/audit", version = "0.11.0" } -pdfcraft-fonts = { path = "crates/fonts", version = "0.11.0" } -pdfcraft-model = { path = "crates/model", version = "0.11.0" } -pdfcraft-content = { path = "crates/content", version = "0.11.0" } -pdfcraft-redact = { path = "crates/redact", version = "0.11.0" } -pdfcraft-print = { path = "crates/print", version = "0.11.0" } -pdfcraft-xfdf = { path = "crates/xfdf", version = "0.11.0" } -pdfcraft-sign = { path = "crates/sign", version = "0.11.0" } -pdfcraft-optimize = { path = "crates/optimize", version = "0.11.0" } -pdfcraft-a11y = { path = "crates/a11y", version = "0.11.0" } -pdfcraft-oracle = { path = "crates/oracle", version = "0.11.0" } -pdfcraft-testkit = { path = "crates/testkit", version = "0.11.0" } -pdfcraft-ocr = { path = "crates/ocr", version = "0.11.0" } -pdfcraft-js = { path = "crates/js", version = "0.11.0" } -pdfcraft-compare = { path = "crates/compare", version = "0.11.0" } -pdfcraft-preflight = { path = "crates/preflight", version = "0.11.0" } -pdfcraft-export = { path = "crates/export", version = "0.11.0" } -pdfcraft-edit = { path = "crates/edit", version = "0.11.0" } -pdfcraft-create = { path = "crates/create", version = "0.11.0" } -pdfcraft-forms = { path = "crates/forms", version = "0.11.0" } -pdfcraft-xfa = { path = "crates/xfa", version = "0.11.0" } -pdfcraft-render = { path = "crates/render", version = "0.11.0" } -pdfcraft-engine = { path = "crates/engine", version = "0.11.0" } -pdfcraft-platform = { path = "crates/platform", version = "0.11.0" } -pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.11.0" } -pdfcraft-automation = { path = "crates/automation", version = "0.11.0", default-features = false } +pdfcraft-geom = { path = "crates/geom", version = "0.12.0" } +pdfcraft-filters = { path = "crates/filters", version = "0.12.0" } +pdfcraft-cos = { path = "crates/cos", version = "0.12.0" } +pdfcraft-crypt = { path = "crates/crypt", version = "0.12.0" } +pdfcraft-organize = { path = "crates/organize", version = "0.12.0" } +pdfcraft-measure = { path = "crates/measure", version = "0.12.0" } +pdfcraft-annot = { path = "crates/annot", version = "0.12.0" } +pdfcraft-audit = { path = "crates/audit", version = "0.12.0" } +pdfcraft-fonts = { path = "crates/fonts", version = "0.12.0" } +pdfcraft-model = { path = "crates/model", version = "0.12.0" } +pdfcraft-content = { path = "crates/content", version = "0.12.0" } +pdfcraft-redact = { path = "crates/redact", version = "0.12.0" } +pdfcraft-print = { path = "crates/print", version = "0.12.0" } +pdfcraft-xfdf = { path = "crates/xfdf", version = "0.12.0" } +pdfcraft-sign = { path = "crates/sign", version = "0.12.0" } +pdfcraft-optimize = { path = "crates/optimize", version = "0.12.0" } +pdfcraft-a11y = { path = "crates/a11y", version = "0.12.0" } +pdfcraft-oracle = { path = "crates/oracle", version = "0.12.0" } +pdfcraft-testkit = { path = "crates/testkit", version = "0.12.0" } +pdfcraft-ocr = { path = "crates/ocr", version = "0.12.0" } +pdfcraft-js = { path = "crates/js", version = "0.12.0" } +pdfcraft-compare = { path = "crates/compare", version = "0.12.0" } +pdfcraft-preflight = { path = "crates/preflight", version = "0.12.0" } +pdfcraft-export = { path = "crates/export", version = "0.12.0" } +pdfcraft-edit = { path = "crates/edit", version = "0.12.0" } +pdfcraft-create = { path = "crates/create", version = "0.12.0" } +pdfcraft-forms = { path = "crates/forms", version = "0.12.0" } +pdfcraft-xfa = { path = "crates/xfa", version = "0.12.0" } +pdfcraft-render = { path = "crates/render", version = "0.12.0" } +pdfcraft-engine = { path = "crates/engine", version = "0.12.0" } +pdfcraft-platform = { path = "crates/platform", version = "0.12.0" } +pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.12.0" } +pdfcraft-automation = { path = "crates/automation", version = "0.12.0", default-features = false } # pdf hayro = "0.7.1" diff --git a/crates/content/Cargo.toml b/crates/content/Cargo.toml index c553df0b8c..e6b4c28a42 100644 --- a/crates/content/Cargo.toml +++ b/crates/content/Cargo.toml @@ -10,3 +10,6 @@ workspace = true [dependencies] pdfcraft-cos = { workspace = true } +# The renderer's content tokenizer, already in the tree through pdfcraft-cos: inline images end +# where the renderer ends them. +hayro-syntax = { workspace = true } diff --git a/crates/content/README.md b/crates/content/README.md index 6ab984dd49..fedeadc56b 100644 --- a/crates/content/README.md +++ b/crates/content/README.md @@ -13,7 +13,10 @@ The parser is tolerant (unreadable bytes are skipped and counted, never fatal) a panics. `redact` and (later) `edit` build their interpreters on it; the graphics-state tracker and the `Patch` API of architecture §6 grow here. -Unfiltered inline images with built-in or Indexed colour spaces, and image masks, use their -sample layout (including per-row byte padding) to locate `EI`. This preserves pixel bytes -that look like the terminator. Filtered images and resource-defined colour spaces retain -the tolerant delimiter heuristic; damaged or truncated layouts fall back to it too. +An inline image ends where the renderer's tokenizer (`hayro-syntax`) ends it, so the parser +and the renderer read the same operators after the image, and redaction removes what the page +actually draws. Only where that tokenizer reads no image (its parameters don't read, or no `EI` +passes its checks) does the parser locate `EI` by its own rules: unfiltered images with built-in +or Indexed colour spaces, and image masks, by their sample layout (including per-row byte +padding), other images by the tolerant delimiter heuristic. Such an image is counted in +`Parsed::ambiguous_images`, because the operators after it may not be the ones the page draws. diff --git a/crates/content/src/lib.rs b/crates/content/src/lib.rs index eb126ffa21..097ec26897 100644 --- a/crates/content/src/lib.rs +++ b/crates/content/src/lib.rs @@ -60,6 +60,10 @@ pub struct Parsed { pub ops: Vec, /// Bytes that could not be read as operands or operators. pub skipped: usize, + /// Inline images the renderer's tokenizer reads no image for (it stops, or finds no `EI` it + /// accepts). Their end here is a guess, so the operators after them may not be the ones the + /// page draws. + pub ambiguous_images: usize, /// For [`Pieces::parse`]: per piece, whether a comment runs into it from the piece before /// and whether one runs out of it into the next. Only boundaries are recorded, so the size /// depends on the number of pieces, not of comments. @@ -142,19 +146,27 @@ fn parse_pieces(data: &[u8], ends: &[usize]) -> Parsed { start.get_or_insert(at); operands.push(Object::Null); } - b"BI" => match inline_image(data, lx.pos) { - Ok((dict, img, end)) => { - lx.pos = end; - out.ops.push(Op { op: b"BI".to_vec(), operands: Vec::new(), inline: Some((dict, img)), span: start.take().unwrap_or(at)..end }); - operands.clear(); + b"BI" => { + let image = inline_image(data, at, lx.pos); + out.ambiguous_images += usize::from(image.ambiguous); + let end = image.end; + lx.pos = end; + match image.image { + Some((dict, img)) => { + out.ops.push(Op { + op: b"BI".to_vec(), + operands: Vec::new(), + inline: Some((dict, img)), + span: start.take().unwrap_or(at)..end, + }); + } + None => { + out.skipped += end.saturating_sub(at); + start = None; + } } - Err(end) => { - out.skipped += end.saturating_sub(at); - lx.pos = end; - start = None; - operands.clear(); - } - }, + operands.clear(); + } _ => { let s = start.take().unwrap_or(at); out.ops.push(Op { op: kw.to_vec(), operands: std::mem::take(&mut operands), inline: None, span: s..lx.pos }); @@ -198,11 +210,29 @@ fn comment_edges(data: &[u8], trivia: std::ops::Range, ends: &[usize], bo } } -/// Read an inline image starting after `BI`: (parameters, data, position after `EI`). `Err` holds -/// the position after the `EI` (or the end of the stream) of an image whose parameters can't be read. -fn inline_image(data: &[u8], pos: usize) -> Result<(Dict, Vec, usize), usize> { +/// An inline image as [`inline_image`] reads it. +struct Inline { + /// Its parameters and data; `None` when the parameters can't be read. + image: Option<(Dict, Vec)>, + /// The position after its `EI` (or the end of the stream). + end: usize, + /// The renderer's tokenizer reads no image here, so `end` is this parser's guess. + ambiguous: bool, +} + +/// Read the inline image whose `BI` keyword starts at `bi` (its parameters start at `pos`). +/// +/// Filtered image data has no length, and `EI` can occur inside it, so where it ends is a +/// heuristic choice among `EI` candidates. The renderer's choice decides which operators the +/// page draws after the image, and redaction must see the same ones, so the end is taken from +/// the renderer's tokenizer (hayro-syntax) rather than worked out here by another rule. Only +/// when that tokenizer reads no image does this parser fall back to its own rules, and the +/// image is marked ambiguous. +fn inline_image(data: &[u8], bi: usize, pos: usize) -> Inline { let mut lx = Lexer::new(data, pos); let mut dict = Dict::new(); + let mut readable = true; + let mut stop = pos; loop { lx.skip_ws(); if lx.eat_keyword(b"ID") { @@ -216,22 +246,68 @@ fn inline_image(data: &[u8], pos: usize) -> Result<(Dict, Vec, usize), usize let Some((k, v)) = entry else { // A stream that ends inside the parameters is cut short, not unreadable: keep what was read. if at >= data.len() { - return Ok((dict, Vec::new(), data.len())); + return Inline { image: Some((dict, Vec::new())), end: data.len(), ambiguous: false }; } - // Without an `EI` after it, the image runs to the end of the stream. - return Err(end_of_image(data, at, b"").map_or(data.len(), |(_, end)| end)); + readable = false; + stop = at; + break; }; dict.set(k, v); } + if let Some((start, end_data, end)) = renderer_image_end(data, bi) { + // Where the renderer ends the data at an `EI` found by its heuristic, one whitespace byte + // before that `EI` separates it from the data. + let mut samples = data.get(start..end_data).unwrap_or_default(); + if data.get(end_data..).is_some_and(|rest| rest.starts_with(b"EI")) + && let Some((last, rest)) = samples.split_last() + && is_ws(*last) + { + samples = rest; + } + return Inline { image: readable.then(|| (dict, samples.to_vec())), end, ambiguous: false }; + } + if !readable { + // Without an `EI` after it, the image runs to the end of the stream. + return Inline { image: None, end: end_of_image(data, stop, b"").map_or(data.len(), |(_, end)| end), ambiguous: true }; + } // One whitespace byte separates ID from the data. let begin = (lx.pos + 1).min(data.len()); let end = unfiltered_image_end(data, begin, &dict) .or_else(|| declared_image_end(data, begin, &dict)) .or_else(|| image_data_end(data, begin, filter_eod(&dict))); - Ok(match end { - Some((end_data, end)) => (dict, data[begin..end_data].to_vec(), end), - None => (dict, data[begin..].to_vec(), data.len()), - }) + let (image, end) = match end { + Some((end_data, end)) => (data.get(begin..end_data).unwrap_or_default().to_vec(), end), + None => (data.get(begin..).unwrap_or_default().to_vec(), data.len()), + }; + Inline { image: Some((dict, image)), end, ambiguous: true } +} + +/// Where the renderer's tokenizer ends the inline image whose `BI` starts at `bi`: the start and +/// end of its data and the position after its `EI`. `None` when it reads no image there (its +/// parameters don't read, or no `EI` passes its checks). +fn renderer_image_end(data: &[u8], bi: usize) -> Option<(usize, usize, usize)> { + use hayro_syntax::content::TypedIter; + use hayro_syntax::content::ops::{InlineImage, TypedInstruction}; + let tail = data.get(bi..)?; + let mut iter = TypedIter::new(tail); + let (start, len) = match iter.next()? { + TypedInstruction::InlineImage(InlineImage(stream)) => match stream.raw_data() { + // The data is a slice of `tail`: its offset is where it starts. + std::borrow::Cow::Borrowed(samples) => ((samples.as_ptr() as usize).checked_sub(tail.as_ptr() as usize)?, samples.len()), + std::borrow::Cow::Owned(_) => return None, + }, + _ => return None, + }; + let end_data = start.checked_add(len)?; + // `committed` is past the whitespace after `EI`; the image ends at the `EI`. + let mut end = iter.committed(); + while end > end_data && tail.get(end - 1).is_some_and(|b| is_ws(*b)) { + end -= 1; + } + if end < end_data.checked_add(2)? || tail.get(end - 2..end) != Some(b"EI") { + return None; + } + Some((bi.checked_add(start)?, bi.checked_add(end_data)?, bi.checked_add(end)?)) } /// The end-of-data marker (`>` for ASCIIHex, `~>` for ASCII85) of the outer stream filter, if any. diff --git a/crates/content/src/tests.rs b/crates/content/src/tests.rs index 094a0665df..c9abef8349 100644 --- a/crates/content/src/tests.rs +++ b/crates/content/src/tests.rs @@ -102,10 +102,11 @@ fn unfiltered_inline_masks_default_to_one_bit_and_indexed_images_have_one_compon let parsed = inline_pixels(parameters.as_bytes(), b" EI "); assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI "); assert_eq!(parsed.ops.len(), 3); - // Indexed samples permit at most 8 bits, even though device spaces permit 16. + // Indexed samples permit at most 8 bits, even though device spaces permit 16, so the + // length is unknown, and the renderer takes the second `EI`. let invalid = parameters.replace("/W 4", "/W 2").replace("/BPC 8", "/BPC 16"); let parsed = inline_pixels(invalid.as_bytes(), b" EI "); - assert!(parsed.ops[1].inline.as_ref().unwrap().1.is_empty()); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI "); } } @@ -127,7 +128,7 @@ fn unfiltered_inline_image_terminator_must_follow_its_samples() { } #[test] -fn filtered_unknown_and_damaged_inline_images_keep_the_existing_fallback() { +fn filtered_unknown_and_damaged_inline_images_end_where_the_renderer_ends_them() { for parameters in [ b"/W 4 /H 1 /BPC 8 /CS /G /F /A85".as_slice(), b"/W 4 /H 1 /BPC 8 /CS /ResourceColor", @@ -142,12 +143,20 @@ fn filtered_unknown_and_damaged_inline_images_keep_the_existing_fallback() { b"/W 9223372036854775807 /H 9223372036854775807 /BPC 16 /CS /CMYK", b"/W 9223372036854775807 /H 2 /BPC 8 /CS /G", ] { + // What follows the first `EI` (`!!~>`) does not read as content, so the renderer takes + // the later one. let parsed = inline_pixels(parameters, b"! EI !!~>"); - assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b"!", "{}", String::from_utf8_lossy(parameters)); + let label = String::from_utf8_lossy(parameters); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b"! EI !!~>", "{label}"); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), [b"q".as_slice(), b"BI", b"Q"], "{label}"); + assert_eq!(parsed.ambiguous_images, 0, "{label}"); } - // A short payload still uses the tolerant recovery, including when there is no EI. - for src in [b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc\nEI Q".as_slice(), b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc"] { - assert_eq!(parse(src).ops[0].inline.as_ref().unwrap().1, b"abc"); + // A short payload is read whole; without an `EI`, the renderer reads no image and this + // parser's end is a guess. + for (src, ambiguous) in [(b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc\nEI Q".as_slice(), 0), (b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc", 1)] { + let parsed = parse(src); + assert_eq!(parsed.ops[0].inline.as_ref().unwrap().1, b"abc"); + assert_eq!(parsed.ambiguous_images, ambiguous); } } @@ -459,3 +468,38 @@ fn ascii85_inline_image_does_not_terminate_at_gt_ei_inside_payload() { assert_eq!(img, b"!!!!>EI !!~>"); } } + +/// The operators the renderer's tokenizer reads from `data`. +fn renderer_operators(data: &[u8]) -> Vec> { + let mut iter = hayro_syntax::content::UntypedIter::new(data); + let mut out = Vec::new(); + while let Some(instruction) = iter.next() { + out.push(instruction.operator.as_ref().to_vec()); + } + out +} + +/// Inline images end where the renderer ends them, so both read the same operators after one: +/// an `EI` without whitespace before it in the data, upper-case text ending in "EI" after the +/// image, and an `EI` inside compressed bytes (#3579). +#[test] +fn inline_images_end_where_the_renderer_ends_them() { + let flate: Vec = (0..FLATE_WITH_EI.len()).step_by(2).map(|i| u8::from_str_radix(&FLATE_WITH_EI[i..i + 2], 16).unwrap()).collect(); + let mut stored = b"q BI /W 64 /H 1 /BPC 8 /CS /G /F /Fl ID ".to_vec(); + stored.extend_from_slice(&flate); + stored.extend_from_slice(b"\nEI Q BT /F1 24 Tf 72 200 Td (AFTER) Tj ET"); + let fixtures: [&[u8]; 6] = [ + b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7FEI 1 0 0 1 200 0 cm EI BT /F1 10 Tf 10 100 Td (SECRET) Tj ET", + b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F> EI BT /F1 10 Tf 10 100 Td (ZWEI) Tj (EIN FREI BEI) Tj ET", + b"q BI /W 2 /H 1 /CS /G /BPC 8 /F /AHx ID 7F7F> EI Q BT (DREI) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 00> EI", + b"BI /W 4 /H 1 /BPC 8 /CS /G /F /A85 ID ! EI !!~>\nEI Q", + b"q BI /W 4 /H 1 /BPC 8 /CS /G ID EI \nEI Q", + &stored, + ]; + for data in fixtures { + let parsed = parse(data); + let ours: Vec> = parsed.ops.iter().map(|op| op.op.clone()).collect(); + assert_eq!(ours, renderer_operators(data), "{}", String::from_utf8_lossy(data)); + assert_eq!((parsed.skipped, parsed.ambiguous_images), (0, 0), "{}", String::from_utf8_lossy(data)); + } +} diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index d3db00fe30..bef515f981 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -17,7 +17,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; use pdfcraft_fonts::{helvetica_width, literal, win_ansi}; #[derive(Debug, thiserror::Error, PartialEq)] @@ -601,7 +601,7 @@ fn isolation(doc: &Document, list: &[Object]) -> Isolation { /// Put a mark's content on a page: behind (with the behind-marks at the start) or on top /// (appended, after wrapping the original content in q/Q). fn place(doc: &mut Document, page: &pdfcraft_model::Page, kind: MarkKind, content: Vec, behind: bool) -> Result<(), EditError> { - place_tagged(doc, page, kind.tag(), content, behind) + place_tagged(doc, page, kind.tag(), content, behind).map(drop) } /// Put content on a page, tagged `tag` (see [`place`]). On top, whatever graphics state the @@ -609,7 +609,8 @@ fn place(doc: &mut Document, page: &pdfcraft_model::Page, kind: MarkKind, conten /// content appended after an earlier PdfCraft wrapper by another tool (#3084): the stretch of /// content that leaves state set is wrapped in q/Q, once. Any known unfinished marked-content /// sequences are closed at the end too, so hidden optional content cannot hide the new mark. -fn place_tagged(doc: &mut Document, page: &pdfcraft_model::Page, tag: &str, content: Vec, behind: bool) -> Result<(), EditError> { +/// Returns the new content stream. +fn place_tagged(doc: &mut Document, page: &pdfcraft_model::Page, tag: &str, content: Vec, behind: bool) -> Result { let mut list = contents(doc, page)?; // The behind-marks at the start: backgrounds in the order added, then watermarks. let behind_kinds = [MarkKind::Background.tag(), MarkKind::Watermark.tag()]; @@ -625,7 +626,8 @@ fn place_tagged(doc: &mut Document, page: &pdfcraft_model::Page, tag: &str, cont Isolation::Unknown { unclosed, marked } => (behind_marks < list.len()).then_some((behind_marks, unclosed, marked)), } }; - let mark = Object::Ref(doc.add(Object::Stream(Stream::flate(tagged(tag), &content)))); + let mark_ref = doc.add(Object::Stream(Stream::flate(tagged(tag), &content))); + let mark = Object::Ref(mark_ref); if behind { // Right after the backgrounds already there, so a new background is painted over the // older ones (#3591) and a watermark over all of them. Not after the watermarks: on a @@ -646,7 +648,7 @@ fn place_tagged(doc: &mut Document, page: &pdfcraft_model::Page, tag: &str, cont list.push(mark); } doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(list)))?; - Ok(()) + Ok(mark_ref) } /// Put content tagged `tag` first inside the q/Q wrapper, so it applies to the page's original @@ -664,7 +666,8 @@ fn place_inside_wrapper(doc: &mut Document, page: &pdfcraft_model::Page, tag: &s /// Draw `content` on top of a page (0-based) as permanent page content, with standard Helvetica /// available as `/PCHelv`. The stream is tagged `tag` (not a mark kind, so Remove never takes /// it away); the original content is wrapped in q/Q first. Used for applied redaction boxes. -pub fn stamp(doc: &mut Document, page: usize, tag: &str, content: Vec) -> Result<(), EditError> { +/// Returns the new content stream. +pub fn stamp(doc: &mut Document, page: usize, tag: &str, content: Vec) -> Result { let all = page_list(doc); check(&[page], all.len())?; add_resources(doc, &all[page], None, Some(&content))?; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 432d313d0c..9e7ca1380c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -987,16 +987,17 @@ fn recovered_render_password(cos: &pdfcraft_cos::Document) -> Option { /// /// Redaction decides what lies under a mark from the positions its own interpreter computes, and /// its verify pass reuses them. Where they differ from the renderer's (a standard font without -/// /Widths, whose advances the interpreter approximates; a CMap, or a font set outside `Tf`, +/// /Widths, whose advances the interpreter approximates; a CMap /// that it reads differently), a mark drawn over the text as displayed lies over glyphs the /// interpreter believes are elsewhere: they stay in the file and nothing notices. The renderer is /// what the user saw when placing the marks, so it is asked. `after` is written as a save would -/// write it, without the marks' own overlay (the box and its label are drawn on purpose), its +/// write it, without the marks' own `overlays` (the box and its label are drawn on purpose), its /// text is extracted, and a glyph still mostly inside a mark fails the edit: the caller keeps /// the document as it was. fn shown_under_redactions( after: &pdfcraft_cos::Document, marks: &[pdfcraft_redact::Mark], + overlays: &[pdfcraft_cos::ObjRef], pages: &[pdfcraft_render::PageInfo], password: Option<&str>, ) -> Result<(), EditError> { @@ -1013,8 +1014,7 @@ fn shown_under_redactions( for &pi in &marked { let Some(page) = model.get(pi) else { continue }; let Some(Object::Array(list)) = page.dict.get(b"Contents").map(|c| (*shown.resolve(c)).clone()) else { continue }; - let kept: Vec = - list.into_iter().filter(|o| shown.resolve(o).as_dict().and_then(|d| d.name(b"PCMark")) != Some(b"Redaction")).collect(); + let kept: Vec = list.into_iter().filter(|o| !o.as_ref().is_some_and(|r| overlays.contains(&r))).collect(); shown.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(kept))).map_err(|e| EditError::Write(e.to_string()))?; } let bytes = write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?; @@ -2374,6 +2374,9 @@ struct EditCtx { /// Fields the edit renamed to keep imported names separate (old → new), for the repair /// log and the undo label. renamed_fields: Vec<(String, String)>, + /// The overlay streams the edit's applied redactions added, which alone the renderer's check + /// leaves out. + redaction_overlays: Vec, } impl EditCtx { @@ -2400,6 +2403,7 @@ impl EditCtx { seed, count: 0, renamed_fields: Vec::new(), + redaction_overlays: Vec::new(), } } @@ -2758,7 +2762,7 @@ fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut EditCtx) -> pdfcraft_edit::update_content(doc, *page, *index, &AddedContent::Image(pdfcraft_edit::AddedImage { image, ..old }))?; } Edit::ApplyRedactions { pages } => { - pdfcraft_redact::apply(doc, pages.as_deref())?; + cx.redaction_overlays = pdfcraft_redact::apply(doc, pages.as_deref())?.overlays; } Edit::ClearRedactions => { pdfcraft_redact::clear_marks(doc, None)?; @@ -3820,6 +3824,7 @@ impl Session { cx.js = Some(js::JsRunner::new(&next, &name)); } cx.renamed_fields.clear(); + cx.redaction_overlays.clear(); // `next` is a copy: if the edit fails or crashes, the document is unchanged. guard(|| run_edit(&mut next, &edit, &mut cx)) .unwrap_or_else(|m| Err(EditError::Invalid(format!("{} failed unexpectedly ({m}); the document was not changed", edit.label()))))?; @@ -3828,7 +3833,7 @@ impl Session { if let Edit::ApplyRedactions { pages } = &edit { let marks: Vec = pdfcraft_redact::marks(&editor.cos).into_iter().filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))).collect(); - guard(|| shown_under_redactions(&next, &marks, &doc.info.pages, doc.password.as_deref())) + guard(|| shown_under_redactions(&next, &marks, &cx.redaction_overlays, &doc.info.pages, doc.password.as_deref())) .unwrap_or_else(|m| Err(EditError::Invalid(format!("{} failed unexpectedly ({m}); the document was not changed", edit.label()))))?; } // Renamed fields are said, not slipped in: a repair-log note (with the caveat that diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 3d347e6492..3012a1f3fe 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -2253,12 +2253,17 @@ fn setting_the_calculation_order_recalculates_and_undoes() { /// One 612×300 page with the content stream `body`. Object 4 is Helvetica; `resources` is what /// the page's /Resources holds. fn one_line_page(resources: &str, body: &str) -> Vec { + one_line_page_with(resources, "", body) +} + +/// [`one_line_page`] with `stream_entries` added to the content stream's dictionary. +fn one_line_page_with(resources: &str, stream_entries: &str, body: &str) -> Vec { let objs = [ "<< /Type /Catalog /Pages 2 0 R >>".to_string(), "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 300] >>".to_string(), format!("<< /Type /Page /Parent 2 0 R /Resources << {resources} >> /Contents 5 0 R >>"), "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_string(), - format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()), + format!("<< /Length {} {stream_entries} >>\nstream\n{body}\nendstream", body.len()), ]; let mut out = b"%PDF-1.7\n".to_vec(); let mut offsets = Vec::new(); @@ -2289,31 +2294,76 @@ fn mark_shown_text(s: &mut Session, id: DocId, needle: &str) { s.apply(id, mark).unwrap(); } -/// Redaction removes text by the positions its own interpreter works out. Here the font and size -/// come from an ExtGState /Font entry (`gs`), which the renderer honours and the interpreter does -/// not model, so it places no glyph under the mark the user drew over the word as displayed: it -/// used to report success and leave the word in the file. The renderer is asked now, and the edit -/// is refused. +/// Redaction removes text by the positions its own interpreter works out, and the renderer has +/// the last word: where the two disagree, a page that still shows a glyph mostly under a mark +/// fails the edit. The check is run here on a page nothing was removed from, as if the +/// interpreter had placed the word elsewhere. Only the overlay streams an apply added are left +/// out: a content stream of the input labelled like one (`/PCMark /Redaction`) is checked too. #[test] -fn redaction_is_refused_when_the_page_still_shows_text_under_a_mark() { +fn the_renderer_check_refuses_text_still_shown_under_a_mark() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "/PCMark /Redaction", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("shown.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let doc = s.get(id).unwrap(); + let mut cos = doc.editor.as_ref().unwrap().cos.clone(); + let marks = pdfcraft_redact::marks(&cos); + let refused = |cos: &pdfcraft_cos::Document| { + let shown = shown_under_redactions(cos, &marks, &[], &doc.info.pages, None); + matches!(shown, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0) + }; + assert!(refused(&cos)); + // The page's contents as an array, as they are once an apply has added its overlay. + let page = pdfcraft_model::pages(&cos).swap_remove(0); + let contents = page.dict.get(b"Contents").cloned().unwrap(); + cos.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), pdfcraft_cos::Object::Array(vec![contents]))).unwrap(); + assert!(refused(&cos), "the labelled input stream was not checked"); +} + +/// Inline image data `7FEI 1 0 0 1 200 0 cm` holds an `EI` the renderer ends the image at, so it +/// runs the `cm` and shows SECRET 200 pt to the right. Redaction reads the image the same way, so +/// a mark drawn over the word as shown takes it out of the file. +#[test] +fn text_after_an_inline_image_is_redacted_where_the_page_shows_it() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page( + "/Font << /F1 4 0 R >>", + "BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7FEI 1 0 0 1 200 0 cm EI BT /F1 10 Tf 10 100 Td (SECRET) Tj ET", + ); + let id = s.open("inline.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET"); + s.apply(id, Edit::ApplyRedactions { pages: None }).unwrap(); + assert!(!page_texts(&s, id).concat().contains("SECRET")); + let saved = s.save_bytes(id).unwrap(); + let held = |d: &[u8]| d.windows(6).any(|w| w == b"SECRET"); + assert!(!held(&saved), "the saved file holds the redacted word"); + let reopened = pdfcraft_cos::Document::open(saved).unwrap(); + assert!( + !reopened.scan_objects().any(|(_, o)| matches!(&*o, pdfcraft_cos::Object::Stream(st) if st.decoded().is_ok_and(|d| held(&d)))), + "a saved stream holds the redacted word" + ); +} + +/// A word whose font and size an ExtGState /Font entry (`gs`) sets is placed where the renderer +/// draws it, so a mark drawn over it as displayed takes it out of the file. +#[test] +fn a_word_in_a_font_set_by_an_extgstate_is_redacted() { let line = "Public words then SECRET42 and more"; let mut s = Session::new().with_clock(|| 1_700_000_000); let by_gs = one_line_page("/ExtGState << /GS1 << /Type /ExtGState /Font [4 0 R 16] >> >>", &format!("BT /GS1 gs 20 150 Td ({line}) Tj ET")); let id = s.open("gs.pdf", None, Arc::new(by_gs), None).unwrap(); mark_shown_text(&mut s, id, "SECRET42"); - let refused = s.apply(id, Edit::ApplyRedactions { pages: None }); - assert!(matches!(refused, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0), "{refused:?}"); - // Nothing was changed: the mark is still pending and the page still shows the line. - assert_eq!(s.get(id).unwrap().redaction_marks(), 1); - assert_eq!(page_texts(&s, id), [line]); - - // The same line with its font set by `Tf`: the mark is applied and the word is gone. - let by_tf = one_line_page("/Font << /F1 4 0 R >>", &format!("BT /F1 16 Tf 20 150 Td ({line}) Tj ET")); - let id = s.open("tf.pdf", None, Arc::new(by_tf), None).unwrap(); - mark_shown_text(&mut s, id, "SECRET42"); s.apply(id, Edit::ApplyRedactions { pages: None }).unwrap(); let shown = page_texts(&s, id).swap_remove(0); assert!(shown.starts_with("Public words then") && shown.ends_with("and more") && !shown.contains("SECRET42"), "{shown}"); + let saved = s.save_bytes(id).unwrap(); + let held = |d: &[u8]| d.windows(8).any(|w| w == b"SECRET42"); + assert!(!held(&saved), "the saved file holds the redacted word"); + let reopened = pdfcraft_cos::Document::open(saved).unwrap(); + assert!( + !reopened.scan_objects().any(|(_, o)| matches!(&*o, pdfcraft_cos::Object::Stream(st) if st.decoded().is_ok_and(|d| held(&d)))), + "a saved stream holds the redacted word" + ); } #[test] diff --git a/crates/forms/src/author.rs b/crates/forms/src/author.rs index f5178a6879..544d0a7f8b 100644 --- a/crates/forms/src/author.rs +++ b/crates/forms/src/author.rs @@ -1280,6 +1280,31 @@ pub fn delete_field(doc: &mut Document, name: &str) -> Result<(), FormError> { } } } + // Out of the calculation order too: a `/CO` entry would keep the field, its value and its + // widgets reachable for a full save. + drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r)) +} + +/// Remove the `/CO` entries `gone` matches, whether the form and the array are held directly or +/// by reference. +fn drop_from_calculation_order(doc: &mut Document, gone: impl Fn(ObjRef) -> bool) -> Result<(), FormError> { + let Some(root) = doc.root() else { return Ok(()) }; + let Some(af_obj) = doc.get(root).as_dict().and_then(|d| d.get(b"AcroForm").cloned()) else { return Ok(()) }; + let Some(mut af) = doc.resolve(&af_obj).as_dict().cloned() else { return Ok(()) }; + let Some(co_obj) = af.get(b"CO").cloned() else { return Ok(()) }; + let co = doc.resolve(&co_obj).as_array().cloned().unwrap_or_default(); + let kept: Vec = co.iter().filter(|o| !o.as_ref().is_some_and(&gone)).cloned().collect(); + if kept.len() == co.len() { + return Ok(()); + } + match (co_obj.as_ref(), af_obj.as_ref()) { + (Some(r), _) if doc.get(r).as_array().is_some() => doc.set(r, Object::Array(kept)), + (_, Some(af_ref)) => doc.update_dict(af_ref, |d| d.set(b"CO".to_vec(), Object::Array(kept)))?, + (_, None) => { + af.set(b"CO".to_vec(), Object::Array(kept)); + doc.update_dict(root, |d| d.set(b"AcroForm".to_vec(), Object::Dict(af)))?; + } + } Ok(()) } diff --git a/crates/redact/README.md b/crates/redact/README.md index 2caf067b5b..cca3d51fdc 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -20,7 +20,8 @@ clear_marks(&mut doc, None)?; // remove marks without app - **images**: fully covered → removed; partly covered → a copy with the covered pixels cleared (8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx). Codecs PdfCraft can't re-encode (DCT, JPX, JBIG2, CCITT) are removed whole (fail-closed); inline images under an - area are removed. Images with separate `/SMask` or `/Mask` data are removed whole; + area are removed. Images with separate `/SMask` or `/Mask` data, and images whose data decodes + to more than twice their declared raster (plus a byte per row and 4 KiB), are removed whole; cleared copies drop `/Alternates` so alternate pixels cannot survive; - **vectors**: covered paths are removed; partly covered paths and shadings are clipped so nothing paints inside the areas; @@ -31,8 +32,13 @@ clear_marks(&mut doc, None)?; // remove marks without app page as a `/PCMark /Redaction` stream. A verification pass re-reads each redacted page and fails the operation if any glyph or inline -image is still under an area. Content that can't be decoded makes the operation fail rather than -leave it unredacted. A Redact annotation with neither a usable `/QuadPoints` array nor a usable +image is still under an area. It checks every content stream the page had, whatever its +dictionary calls it; only the overlay streams this `apply` added (`Report::overlays`) are skipped. +Text is placed with the font and size set by `Tf` or by an ExtGState's `/Font` through `gs`, and a +form XObject starts from the text state of the content that draws it. Content that can't be +decoded makes the operation fail rather than leave it unredacted, and so does an inline image the +renderer's tokenizer reads no image for (`RedactError::Unreadable`): where the page's operators +resume after it would be a guess. A Redact annotation with neither a usable `/QuadPoints` array nor a usable `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. @@ -50,15 +56,19 @@ as Sanitize drops them, so a full rewrite of a tagged PDF can't keep a removed c A tree where that can't be done (an element written in place, or one too large to walk) makes `apply` fail with `RedactError::Tags` rather than leave the annotation reachable. -Replacement descriptions are removed only from marked-content spans changed by cleanup. -Unchanged spans retain their descriptions; unused property entries are retired during redaction +Replacement descriptions (`/Alt`, `/ActualText`, `/E`) are removed from marked-content spans +changed by cleanup, from every structure element above them (found through both `/P` and `/K`), +and from elements that lose an annotation reference, which also lose `/T`. An element written in +place above one whose descriptions go makes `apply` fail with `RedactError::Tags`. Unchanged +spans retain their descriptions; unused property entries are retired during redaction and hidden-content cleanup, including aliases in shared/inherited resources. Descriptions still used by surviving pages or forms are kept. A redacted page that inherits `/Resources` from the page tree gets them pinned (by reference; a direct inherited dictionary is made one shared object first) before it is rewritten. The parent's `/Resources` is then retired unless another page still inherits it, so it cannot keep a removed -XObject reachable. Pages that aren't redacted are left untouched. +XObject reachable. Surviving form XObjects lose the resource entries for removed XObjects their own +content doesn't draw, for the same reason. Pages that aren't redacted are left untouched. ## Remove hidden information / Sanitize diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 9a19ba2f0b..601700f257 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -77,7 +77,11 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]] let bytes = row.checked_mul(h)?; let row_bits = row.checked_mul(8)?; bytes.checked_mul(8)?; - let mut data = s.decoded().ok()?; + // Decode no further than the raster can need: a filter stage may legitimately produce up to + // twice it (ASCIIHex text under Flate) plus a PNG predictor's byte per row. Data past that + // can only be a decompression bomb; the image is then removed whole. + let allowance = bytes.checked_add(h)?.checked_mul(2)?.checked_add(4096)?; + let mut data = s.decoded_within(allowance).ok()?; if data.len() < bytes { return None; } diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index a608568f38..b4186fd4e9 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -74,6 +74,10 @@ pub(crate) struct Output { /// Apply mode met vertical-writing text (which it cannot remove) while a redaction region was /// in force: the operation must fail closed rather than leave that text under the mark. pub vertical: bool, + /// The content holds an inline image the renderer reads no image for + /// ([`pdfcraft_content::Parsed::ambiguous_images`]): what this scope placed after it may not + /// be what the page shows, so a redaction must not trust it. + pub ambiguous: bool, pub tag_changes: Vec<(ObjRef, ObjRef, HashSet, HashSet)>, pub error: Option, } @@ -529,8 +533,9 @@ pub(crate) fn unlink_doomed(doc: &mut Document, doomed: &[ObjRef], retired_prope } } // A surviving form can own the same old resource dictionary as a rewritten page. Its - // unused descriptions remain reachable too, even after every page holder is cleaned. - unlink_form_properties(doc, &holder_nodes, retired_properties)?; + // unused descriptions and doomed XObjects remain reachable too, even after every page + // holder is cleaned. + unlink_from_forms(doc, &holder_nodes, &doomed, retired_properties)?; Ok(()) } @@ -567,8 +572,11 @@ fn prune_retired_properties(doc: &Document, res: &mut Dict, names: &[Vec], c Ok(pruned) } -fn unlink_form_properties(doc: &mut Document, holders: &[ObjRef], retired: &[Object]) -> Result<(), CosError> { - if retired.is_empty() { +/// Unlink retired properties and `doomed` XObjects from the resources of the forms reachable from +/// the page-tree `holders`, each unless the form's own content (or a form inheriting its +/// resources) still uses the name. Fails closed when that cannot be determined. +fn unlink_from_forms(doc: &mut Document, holders: &[ObjRef], doomed: &HashSet, retired: &[Object]) -> Result<(), CosError> { + if retired.is_empty() && doomed.is_empty() { return Ok(()); } let mut pending: Vec = holders @@ -593,7 +601,27 @@ fn unlink_form_properties(doc: &mut Document, holders: &[ObjRef], retired: &[Obj let Some(mut res) = form.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()) else { continue }; let names: Vec> = res_dict(doc, &res, b"Properties").iter().filter(|(_, value)| retired.contains(value)).map(|(name, _)| name.to_vec()).collect(); - if !names.is_empty() && prune_retired_properties(doc, &mut res, &names, &[vec![form.decoded()?]])? { + let mut xo = res_dict(doc, &res, b"XObject"); + let doomed_names: Vec> = + xo.iter().filter(|(_, v)| v.as_ref().is_some_and(|r| doomed.contains(&r))).map(|(n, _)| n.to_vec()).collect(); + if names.is_empty() && doomed_names.is_empty() { + pending.push(res); + continue; + } + let data = form.decoded()?; + let mut pruned = false; + for name in doomed_names { + match content_draws(doc, std::slice::from_ref(&data), &xo, &name, 0, &mut HashSet::new()) { + Some(true) => {} + Some(false) => pruned |= xo.remove(&name).is_some(), + None => return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect shared form resources".into() }), + } + } + if pruned { + res.set(b"XObject".to_vec(), Object::Dict(xo)); + } + pruned |= prune_retired_properties(doc, &mut res, &names, &[vec![data]])?; + if pruned { doc.update_dict(reference, |d| d.set(b"Resources".to_vec(), Object::Dict(res.clone())))?; } pending.push(res); @@ -690,6 +718,34 @@ fn prune_form_resources(doc: &Document, res: &mut Dict, content: &[Vec]) -> Ok(()) } +/// The font, writing direction and size an ExtGState's `/Font [font size]` entry sets, as `Tf` +/// would (ISO 32000-2 §8.4.5); `None` when `name` sets no font. The metrics are cached with the +/// fonts `Tf` names, under a key no resource name can take (a name cannot hold a null byte). +fn extgstate_font( + doc: &Document, + scope: &mut Scope<'_>, + ext_gstates: &Dict, + name: &[u8], + fallback: &Rc, +) -> Option<(Rc, bool, f64)> { + let state = doc.resolve(ext_gstates.get(name)?).as_dict().cloned()?; + let entry = doc.resolve(state.get(b"Font")?).as_array().cloned()?; + let [font, size] = entry.as_slice() else { return None }; + let size = doc.resolve(size).as_f64()?; + let font_dict = doc.resolve(font).as_dict().cloned(); + let vertical = font_dict.as_ref().is_some_and(|d| is_vertical(doc, d)); + let key = [b"\0".as_slice(), name].concat(); + let metrics = match scope.fonts.get(&key) { + Some(m) => m.clone(), + None => { + let m = font_dict.as_ref().map(|d| Rc::new(Metrics::from_dict(doc, d))).unwrap_or_else(|| fallback.clone()); + scope.fonts.insert(key, m.clone()); + m + } + }; + Some((metrics, vertical, size)) +} + /// One glyph of a shown string. struct Glyph { bytes: std::ops::Range, @@ -700,11 +756,9 @@ struct Glyph { /// Process a scope's content streams (state flows from one stream to the next). pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], resources: &Dict, ctm: Matrix) -> Output { - let mut out = Output::default(); - let fallback = Rc::new(Metrics::fallback()); - let mut gs = Gs { + let gs = Gs { ctm, - font: fallback.clone(), + font: Rc::new(Metrics::fallback()), vertical: false, size: 0.0, char_spacing: 0.0, @@ -714,9 +768,19 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< leading: 0.0, rise: 0.0, }; + process_from(doc, scope, streams, resources, gs) +} + +/// [`process`] from graphics state `gs`: a form XObject starts with its caller's, text state +/// included (ISO 32000-2 §8.10.1), so text it shows without setting a font of its own is placed +/// with the caller's font, size and spacing, as the renderer places it. +fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], resources: &Dict, mut gs: Gs) -> Output { + let mut out = Output::default(); + let fallback = Rc::new(Metrics::fallback()); let mut stack: Vec = Vec::new(); let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); let fonts_res = res_dict(doc, resources, b"Font"); + let ext_gstates = res_dict(doc, resources, b"ExtGState"); let mut xobjects = res_dict(doc, resources, b"XObject"); // The current path: its operators, its bounding box in user space, and whether it clips. let mut path: Vec = Vec::new(); @@ -729,6 +793,7 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< // next. Parse them joined, then give each operator back to the piece its keyword is in. let joined = Pieces::join(streams); let mut parsed = joined.parse(); + out.ambiguous = parsed.ambiguous_images > 0; // A rewritten piece loses its comments, so keep where they began and ended: a comment that // ran on into the next piece still hides that piece's first line. let comment_edges = joined.comment_edges(&parsed); @@ -865,6 +930,11 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< }; } } + b"gs" => { + if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &ext_gstates, n, &fallback)) { + (gs.font, gs.vertical, gs.size) = (font, vertical, size); + } + } b"Tc" => gs.char_spacing = op.num(0).unwrap_or(0.0), b"Tw" => gs.word_spacing = op.num(0).unwrap_or(0.0), b"Tz" => gs.scale = op.num(0).unwrap_or(100.0) / 100.0, @@ -1223,11 +1293,12 @@ fn xobject( let res = own.clone().unwrap_or_else(|| resources.clone()); let saved_fonts = std::mem::take(&mut scope.fonts); scope.depth += 1; - let inner = process(doc, scope, std::slice::from_ref(&data), &res, fctm); + let inner = process_from(doc, scope, std::slice::from_ref(&data), &res, Gs { ctm: fctm, ..gs.clone() }); scope.depth -= 1; scope.fonts = saved_fonts; out.residue += inner.residue; out.vertical |= inner.vertical; + out.ambiguous |= inner.ambiguous; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); out.tag_changes.extend(inner.tag_changes); diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 968773b91d..e67b373a11 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -143,6 +143,10 @@ pub struct Report { /// The form was an AcroForm+XFA form and its `/XFA` packets (which still held redacted field /// values, and from which a relayout could rebuild them) were removed. pub xfa_removed: bool, + /// The content streams this apply added to draw the marks' boxes and labels. They are the + /// only streams a check for text left under the marks may skip: an input stream labelled + /// like an overlay (`/PCMark /Redaction`) is checked like any other. + pub overlays: Vec, } fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { @@ -407,6 +411,12 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result) -> Result) -> Result = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); let (list, data) = page_streams(doc, &page.dict, pi)?; - // The overlay stream (last) draws text of its own: the overlay label is allowed. - let original: Vec> = list - .iter() - .zip(data) - .filter(|(o, _)| doc.resolve(o).as_dict().and_then(|d| d.name(b"PCMark")) != Some(b"Redaction")) - .map(|(_, d)| d) - .collect(); + // The overlay stream draws text of its own: the overlay label is allowed. + let original: Vec> = + list.iter().zip(data).filter(|(o, _)| !o.as_ref().is_some_and(|r| report.overlays.contains(&r))).map(|(_, d)| d).collect(); let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); let mut scratch = Report::default(); let mut scope = Scope::new(&rects, Mode::Verify, &mut scratch); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index ab5735e065..60c158b180 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -6,15 +6,49 @@ //! references to marked content that is now empty are dropped from the element. use std::collections::{HashMap, HashSet}; +use std::ops::Range; use pdfcraft_cos::{Dict, Document, ObjRef, Object}; use crate::RedactError; -/// Each MCID's operators (serialized), and whether anything is still drawn inside it. -fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> HashMap, bool)> { - let mut out: HashMap, bool)> = HashMap::new(); - let mut stack: Vec> = Vec::new(); +/// The marked content of one content stream. Every operator inside some MCID is serialized once +/// into `ops`, and each MCID holds the ranges of `ops` its sequences span (marked-content +/// operators are not written, so a sequence is one range however deeply it nests), with whether +/// anything is drawn inside it. Copying the operators into every enclosing MCID instead would +/// hold `depth` copies of nested content. +#[derive(Default)] +struct Marked { + ops: Vec, + ids: HashMap>, bool)>, +} + +impl Marked { + /// The bytes of `ranges`, which are ranges of `ops`. + fn bytes<'a>(&'a self, ranges: &'a [Range]) -> impl Iterator + 'a { + ranges.iter().flat_map(|r| self.ops.get(r.clone()).unwrap_or_default()) + } +} + +/// Most bytes [`touched`] compares for one stream. Nested sequences share their operators, so the +/// comparisons can add up to the content's size times its nesting depth; past this, the MCIDs +/// left are taken as changed (their descriptions go) rather than compared. +const MAX_COMPARED: usize = 1 << 28; + +fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { + let mut out = Marked::default(); + // Open sequences: the MCID with where its content starts in `ops` and the drawing operators + // written before it, or `None` for a sequence without one. + let mut stack: Vec> = Vec::new(); + let (mut open_ids, mut drawn) = (0usize, 0usize); + let close = |out: &mut Marked, open: Option<(i64, usize, usize)>, drawn: usize| { + if let Some((id, start, drawn_before)) = open { + let end = out.ops.len(); + let e = out.ids.entry(id).or_default(); + e.0.push(start..end); + e.1 |= drawn > drawn_before; + } + }; for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { @@ -28,9 +62,10 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> HashMap None, }; if let Some(id) = id { - out.entry(id).or_default(); + out.ids.entry(id).or_default(); + open_ids += 1; } - stack.push(id); + stack.push(id.map(|id| (id, out.ops.len(), drawn))); continue; } b"BMC" => { @@ -38,11 +73,17 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> HashMap { - stack.pop(); + if let Some(open) = stack.pop() { + open_ids = open_ids.saturating_sub(usize::from(open.is_some())); + close(&mut out, open, drawn); + } continue; } _ => {} } + if open_ids == 0 { + continue; + } let draws = match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().and_then(Object::as_string).is_some_and(|s| !s.bytes.is_empty()), b"TJ" => { @@ -51,13 +92,12 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> HashMap true, _ => false, }; - let mut bytes = Vec::new(); - pdfcraft_content::write_op(&op, &mut bytes); - for id in stack.iter().flatten() { - let e = out.entry(*id).or_default(); - e.0.extend_from_slice(&bytes); - e.1 |= draws; - } + pdfcraft_content::write_op(&op, &mut out.ops); + drawn += usize::from(draws); + } + // Sequences the stream leaves open run to its end. + while let Some(open) = stack.pop() { + close(&mut out, open, drawn); } out } @@ -67,10 +107,17 @@ pub(crate) fn touched(doc: &Document, resources: &Dict, before: &[u8], after: &[ let (b, a) = (by_mcid(doc, resources, before), by_mcid(doc, resources, after)); let mut changed = HashSet::new(); let mut empty = HashSet::new(); - for (id, (ops, _)) in &b { - match a.get(id) { + let mut budget = MAX_COMPARED; + let len = |ranges: &[Range]| ranges.iter().map(|r| r.len()).fold(0usize, usize::saturating_add); + for (id, (old, _)) in &b.ids { + match a.ids.get(id) { Some((new, draws)) => { - if new != ops { + let n = len(old); + let same = n == len(new) && n <= budget && { + budget -= n; + b.bytes(old).eq(a.bytes(new)) + }; + if !same { changed.insert(*id); } if !draws { @@ -136,20 +183,21 @@ pub(crate) fn clean_within( // also from the parent tree and the ID tree: one listed only there still owns marked content // (#3302). The stack pops /K first, so an element reached through /K gets the page it inherits // from its ancestors before a tree reaches it without one. - let mut todo: Vec<(Object, Option)> = - [&b"IDTree"[..], b"ParentTree", b"K"].iter().filter_map(|key| root.get(key).cloned()).map(|k| (k, None)).collect(); + let mut todo: Vec<(Object, Option, Above)> = + [&b"IDTree"[..], b"ParentTree", b"K"].iter().filter_map(|key| root.get(key).cloned()).map(|k| (k, None, Above::default())).collect(); let mut seen = HashSet::new(); let mut pages = HashMap::new(); + let mut parents = Parents::new(); let mut hits: Vec<(ObjRef, Dict, Option, bool)> = Vec::new(); // Whether marked content `x` (a kid on page `pg`) is one this cleanup must change. let touches = |doc: &Document, x: &Object, pg: Option| { mcid_of(doc, x, pg, page, stream).is_some_and(|m| stream.is_some() || changed.contains(&m) || empty.contains(&m)) }; - while let Some((k, pg)) = todo.pop() { + while let Some((k, pg, above)) = todo.pop() { match k { - Object::Array(a) => todo.extend(a.into_iter().map(|x| (x, pg))), + Object::Array(a) => todo.extend(a.into_iter().map(|x| (x, pg, above))), // A number or name tree node written in place. - Object::Dict(d) if !d.contains(b"S") => todo.extend(tree_node_entries(&d).into_iter().map(|x| (x, pg))), + Object::Dict(d) if !d.contains(b"S") => todo.extend(tree_node_entries(&d).into_iter().map(|x| (x, pg, above))), // An element written in place: its kids are followed, but if it owns marked content // this cleanup changes, only rewriting what holds it could change it. Object::Dict(d) => { @@ -161,9 +209,11 @@ pub(crate) fn clean_within( if kids.iter().any(|x| touches(doc, x, pg)) { return Err(RedactError::Tags); } - todo.extend(kids.into_iter().map(|x| (x, pg))); + let above = above.through(&d); + todo.extend(kids.into_iter().map(|x| (x, pg, above))); } Object::Ref(r) => { + parents.record(r, above); if !seen.insert(r) { continue; } @@ -172,13 +222,13 @@ pub(crate) fn clean_within( } let object = doc.get(r); if let Object::Array(a) = &*object { - todo.extend(a.iter().cloned().map(|x| (x, pg))); + todo.extend(a.iter().cloned().map(|x| (x, pg, above))); continue; } let Some(d) = object.as_dict().cloned() else { continue }; if !d.contains(b"S") { // A parent tree or ID tree node: its values may be elements. - todo.extend(tree_node_entries(&d).into_iter().map(|x| (x, pg))); + todo.extend(tree_node_entries(&d).into_iter().map(|x| (x, pg, above))); continue; } let pg = match d.get(b"Pg").and_then(Object::as_ref).or(pg) { @@ -191,11 +241,12 @@ pub(crate) fn clean_within( if sensitive || remap { hits.push((r, d.clone(), pg, sensitive)); } - todo.extend(kids.into_iter().map(|x| (x, pg))); + todo.extend(kids.into_iter().map(|x| (x, pg, Above::element(r)))); } _ => {} } } + let sensitive: Vec = hits.iter().filter(|h| h.3).map(|h| h.0).collect(); let mut n = 0; for (r, d, pg, sensitive) in hits { let kids = d.get(b"K").map(|k| doc.resolve(k)); @@ -244,9 +295,88 @@ pub(crate) fn clean_within( })?; n += usize::from(sensitive && doc.get(r).as_dict() != Some(&d)); } + // An element above may describe the changed content too (`/ActualText` on a span's parent + // covers the span, §14.9.4). + n += clear_ancestors(doc, &sensitive, &parents, limit)?; Ok(n) } +/// Where the walk reached an object from: the nearest element above it that is an object of its +/// own, and whether an element written in place between them carries a description (which only +/// rewriting the object holding it could remove). +#[derive(Clone, Copy, Default)] +struct Above { + element: Option, + described_in_place: bool, +} + +impl Above { + fn element(r: ObjRef) -> Self { + Self { element: Some(r), described_in_place: false } + } + + /// Below an element written in place. + fn through(self, d: &Dict) -> Self { + Self { described_in_place: self.described_in_place || DESCRIPTIONS.iter().any(|k| d.contains(k)), ..self } + } +} + +/// The keys of an element that describe its content. +const DESCRIPTIONS: [&[u8]; 3] = [b"Alt", b"ActualText", b"E"]; + +/// Each element's parents as the walk found them: through `/K`, which `/P` may not agree with. +struct Parents(HashMap>); + +impl Parents { + fn new() -> Self { + Self(HashMap::new()) + } + + fn record(&mut self, r: ObjRef, above: Above) { + if above.element.is_some() || above.described_in_place { + self.0.entry(r).or_default().push(above); + } + } +} + +/// Remove the descriptions of every element above `elements`, through both `/P` and the walk's +/// `parents`; returns how many elements changed. [`RedactError::Tags`] when an element written in +/// place above them carries one, or when there are more than `limit` ancestors. +fn clear_ancestors(doc: &mut Document, elements: &[ObjRef], parents: &Parents, limit: usize) -> Result { + let mut todo: Vec = elements.to_vec(); + let mut seen: HashSet = elements.iter().copied().collect(); + let mut cleared = 0; + while let Some(r) = todo.pop() { + let mut up: Vec = doc.get(r).as_dict().and_then(|d| d.get(b"P")).and_then(Object::as_ref).into_iter().collect(); + for above in parents.0.get(&r).into_iter().flatten() { + if above.described_in_place { + return Err(RedactError::Tags); + } + up.extend(above.element); + } + for a in up { + if !seen.insert(a) { + continue; + } + if seen.len() > limit { + return Err(RedactError::Tags); + } + // The structure tree root (no /S) ends the climb. + let Some(d) = doc.get(a).as_dict().filter(|d| d.contains(b"S")).cloned() else { continue }; + if DESCRIPTIONS.iter().any(|k| d.contains(k)) { + doc.update_dict(a, |e| { + for key in DESCRIPTIONS { + e.remove(key); + } + })?; + cleared += 1; + } + todo.push(a); + } + } + Ok(cleared) +} + /// The page an element inherits (§14.7.4.4): the nearest `/Pg` among its ancestors, following /// `/P`. `Ok(None)` when none has one; [`RedactError::Tags`] when the lookup gives up (more than /// [`MAX_PARENTS`] ancestors, or a loop), which isn't the same as having no page. `pages` keeps @@ -337,12 +467,14 @@ pub(crate) fn drop_object_refs_within(doc: &mut Document, gone: &HashSet }; // Elements are reached from the root's /K, and also from the parent tree and the ID tree: // one listed only there (left behind by an editor, say) still points at its annotation. - let mut todo: Vec = [&b"K"[..], b"ParentTree", b"IDTree"].iter().filter_map(|key| root.get(key).cloned()).collect(); + let mut todo: Vec<(Object, Above)> = + [&b"K"[..], b"ParentTree", b"IDTree"].iter().filter_map(|key| root.get(key).cloned()).map(|k| (k, Above::default())).collect(); let mut seen = HashSet::new(); + let mut parents = Parents::new(); let mut hits: Vec = Vec::new(); - while let Some(k) = todo.pop() { + while let Some((k, above)) = todo.pop() { match k { - Object::Array(a) => todo.extend(a), + Object::Array(a) => todo.extend(a.into_iter().map(|x| (x, above))), // A structure element written in place: its kids are followed; one that tags a // removed annotation can't be changed here (see above). Object::Dict(d) if d.contains(b"S") => { @@ -350,11 +482,13 @@ pub(crate) fn drop_object_refs_within(doc: &mut Document, gone: &HashSet if kids.iter().any(|x| refers_to(doc, x, gone)) { return Err(RedactError::Tags); } - todo.extend(kids); + let above = above.through(&d); + todo.extend(kids.into_iter().map(|x| (x, above))); } // A number or name tree node written in place (§7.9.6, §7.9.7): its values and kids. - Object::Dict(d) => todo.extend(tree_node_entries(&d)), + Object::Dict(d) => todo.extend(tree_node_entries(&d).into_iter().map(|x| (x, above))), Object::Ref(r) => { + parents.record(r, above); if !seen.insert(r) { continue; } @@ -363,26 +497,26 @@ pub(crate) fn drop_object_refs_within(doc: &mut Document, gone: &HashSet } let object = doc.get(r); if let Object::Array(a) = &*object { - todo.extend(a.iter().cloned()); + todo.extend(a.iter().cloned().map(|x| (x, above))); continue; } let Some(d) = object.as_dict() else { continue }; if !d.contains(b"S") { // A parent tree or ID tree node: its values may be elements. - todo.extend(tree_node_entries(d)); + todo.extend(tree_node_entries(d).into_iter().map(|x| (x, above))); continue; } let kids = kids_of(doc, d); if kids.iter().any(|x| refers_to(doc, x, gone)) { hits.push(r); } - todo.extend(kids); + todo.extend(kids.into_iter().map(|x| (x, Above::element(r)))); } _ => {} } } let mut n = 0; - for r in hits { + for &r in &hits { let Some(d) = doc.get(r).as_dict().cloned() else { continue }; let kids = d.get(b"K").map(|k| doc.resolve(k)); let array = kids.as_ref().is_some_and(|k| matches!(&**k, Object::Array(_))); @@ -402,14 +536,21 @@ pub(crate) fn drop_object_refs_within(doc: &mut Document, gone: &HashSet } else { e.remove(b"K"); } - // An element left with nothing to tag keeps no text describing what it tagged: - // a link's alternate text often repeats the URL, an annotation's its comment. - if keep.is_empty() { - for key in [&b"Alt"[..], b"ActualText", b"T", b"E"] { - e.remove(key); - } + // An element that lost an annotation keeps no text describing it, even with other + // kids left: a link's alternate text often repeats the URL, an annotation's its + // comment. Elements above it may describe it too. + for key in [&b"Alt"[..], b"ActualText", b"T", b"E"] { + e.remove(key); } })?; } + clear_ancestors(doc, &hits, &parents, limit)?; Ok(n) } + +/// Bytes held for comparing marked content, for tests that bound it. +#[cfg(test)] +pub(crate) fn retained_for_comparison(doc: &Document, resources: &Dict, data: &[u8]) -> usize { + let marked = by_mcid(doc, resources, data); + marked.ops.len() + marked.ids.values().map(|(ranges, _)| ranges.len() * std::mem::size_of::>()).sum::() +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index a9902495df..e35d4f907d 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -2461,7 +2461,12 @@ fn unused_descriptions_in_shared_or_inherited_resources_are_retired() { } } +mod descriptions; +mod inline_images; +mod limits; mod property_retirement; +mod surviving_references; +mod text_state; /// `[(FIRSTARR)] [(LASTARR)] TJ` is malformed (a surplus array operand), but the operator still /// draws only the one immediately before it (LASTARR, 7 glyphs, 35 pt wide from x = 10) -- not diff --git a/crates/redact/src/tests/descriptions.rs b/crates/redact/src/tests/descriptions.rs new file mode 100644 index 0000000000..df6b8555dc --- /dev/null +++ b/crates/redact/src/tests/descriptions.rs @@ -0,0 +1,64 @@ +//! Descriptions in the structure tree (alternate text, actual text, expansions) go with the +//! content they describe, on the element that owns it and on every element above it. + +use super::surviving_references::full_save_holds; +use super::*; + +/// A tagged page: "SECRET" (MCID 0) inside a section whose descriptions repeat it, and "Public" +/// (MCID 1). `paragraph_parent` is the secret paragraph's /P. +fn sectioned(paragraph_parent: &str) -> Document { + let content = b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC /P <> BDC BT /F1 10 Tf 10 100 Td (Public) Tj ET EMC"; + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [11 0 R 10 0 R] >>".to_vec(), + format!("<< /S /P /P {paragraph_parent} /Pg 3 0 R /K 0 >>").into_bytes(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (Public) /K 1 >>".to_vec(), + b"<< /S /Sect /P 8 0 R /ActualText (LEAK_ACTUAL) /Alt (LEAK_ALT) /E (LEAK_EXPANSION) /K 9 0 R >>".to_vec(), + ]) +} + +#[test] +fn an_ancestor_describing_redacted_content_loses_its_descriptions() { + // The paragraph names its section as parent, or (wrongly) the document: the section is found + // either way, by its /K. + for parent in ["11 0 R", "8 0 R"] { + let mut doc = sectioned(parent); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, None).unwrap(); + assert!(!full_save_holds(&doc, b"LEAK_"), "a description of the redacted word survived (paragraph /P {parent})"); + let public = doc.get(ObjRef::new(10, 0)).as_dict().cloned().unwrap(); + assert!(public.contains(b"ActualText"), "an element beside it keeps its own"); + } +} + +#[test] +fn a_link_element_that_keeps_its_text_loses_the_description_of_its_removed_annotation() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /MarkInfo << /Marked true >> /StructTreeRoot 8 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] /StructParents 0 >>" + .to_vec(), + stream("", b"/Link << /MCID 0 >> BDC BT /F1 10 Tf 10 230 Td (See the site) Tj ET EMC"), + FONT.replace("95 0 R", "12 0 R").into_bytes(), + b"<< /Type /Annot /Subtype /Link /Rect [10 228 80 240] /StructParent 1 /A << /S /URI /URI (https://example.org/) >> >>".to_vec(), + b"null".to_vec(), + b"<< /Type /StructTreeRoot /K 9 0 R /ParentTree << /Nums [0 [11 0 R] 1 11 0 R] >> /ParentTreeNextKey 2 >>".to_vec(), + b"<< /Type /StructElem /S /Document /P 8 0 R /K 10 0 R >>".to_vec(), + b"<< /Type /StructElem /S /P /P 9 0 R /ActualText (See LEAK_PARENT) /K 11 0 R >>".to_vec(), + // The link element tags its text and the annotation; its descriptions repeat the address. + b"<< /Type /StructElem /S /Link /P 10 0 R /Pg 3 0 R /Alt (LEAK_ALT) /ActualText (LEAK_ACTUAL) /E (LEAK_E) /T (LEAK_T) /K [0 << /Type /OBJR /Obj 6 0 R /Pg 3 0 R >>] >>".to_vec(), + widths(), + ]); + let done = remove_hidden(&mut doc, &[Hidden::LinksActionsScripts]).unwrap(); + assert!(done.contains(&(Hidden::LinksActionsScripts, 1)), "{done:?}"); + assert!(!full_save_holds(&doc, b"LEAK_"), "a description of the removed link survived"); + let link = doc.get(ObjRef::new(11, 0)).as_dict().cloned().unwrap(); + assert_eq!(link.get(b"K"), Some(&Object::Array(vec![Object::Int(0)])), "the element keeps its text"); +} diff --git a/crates/redact/src/tests/inline_images.rs b/crates/redact/src/tests/inline_images.rs new file mode 100644 index 0000000000..77fd4dc2d7 --- /dev/null +++ b/crates/redact/src/tests/inline_images.rs @@ -0,0 +1,34 @@ +//! Inline image data ends where the renderer ends it, so the operators after it are the ones the +//! page draws. + +use super::*; + +/// The image data `7FEI 1 0 0 1 200 0 cm` holds an `EI` straight after `7F`. The renderer ends +/// the image there (text follows, so what comes after reads as content) and runs the `cm`, +/// drawing SECRET at x = 210, not 10. +const FAKE_EI: &[u8] = b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7FEI 1 0 0 1 200 0 cm EI BT /F1 10 Tf 10 100 Td (SECRET) Tj ET"; + +#[test] +fn text_after_an_inline_image_is_redacted_where_the_renderer_draws_it() { + let mut doc = one_page(FAKE_EI, "", vec![]); + assert_eq!(under(&mut doc, 0, &[[5.0, 95.0, 45.0, 110.0]]), 0, "nothing is drawn at x = 10"); + mark(&mut doc, 0, &[[205.0, 95.0, 245.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("SECRET")); +} + +#[test] +fn an_inline_image_with_one_possible_end_is_redacted() { + let mut doc = one_page(b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F> EI BT /F1 10 Tf 10 100 Td (SECRET) Tj ET", "", vec![]); + mark(&mut doc, 0, &[[5.0, 95.0, 45.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("SECRET")); +} + +/// Without an `EI` the renderer accepts, where the operators resume is a guess: refused. +#[test] +fn a_page_whose_inline_image_the_renderer_cannot_end_is_refused() { + let mut doc = one_page(b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F", "", vec![]); + mark(&mut doc, 0, &[[5.0, 95.0, 45.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} diff --git a/crates/redact/src/tests/limits.rs b/crates/redact/src/tests/limits.rs new file mode 100644 index 0000000000..8519e08608 --- /dev/null +++ b/crates/redact/src/tests/limits.rs @@ -0,0 +1,55 @@ +//! Work and memory stay bounded by what the page declares, whatever the file holds. + +use super::*; + +/// A half-covered 4×1 grey image whose Flate data decodes to `decoded`. +fn half_covered_image(decoded: &[u8]) -> Report { + let dict = "/Type /XObject /Subtype /Image /Width 4 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + let mut doc = one_page(b"q 40 0 0 10 100 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(dict, b"")]); + let Object::Stream(s) = &*doc.get(ObjRef::new(7, 0)) else { panic!() }; + doc.set(ObjRef::new(7, 0), Object::Stream(Stream::flate(s.dict.clone(), decoded))); + mark(&mut doc, 0, &[[95.0, 95.0, 120.0, 115.0]], ""); + apply(&mut doc, None).unwrap() +} + +#[test] +fn an_image_decoding_far_past_its_raster_is_removed_without_decoding_it_all() { + let r = half_covered_image(&[255; 4]); + assert_eq!((r.images_removed, r.images_cleared), (0, 1), "{r:?}"); + // 4 bytes of raster, then 4 MiB the image never uses: decoding stops at a bound derived from + // the raster, and the image goes whole. + let mut bomb = vec![255; 4]; + bomb.resize(4 << 20, 0); + let r = half_covered_image(&bomb); + assert_eq!((r.images_removed, r.images_cleared), (1, 0), "{r:?}"); +} + +/// Marked content nested `depth` deep, each level with its own MCID, around `depth` drawing +/// operators. +fn nested_marked_content(depth: usize) -> Vec { + let mut data = Vec::new(); + for id in 0..depth { + data.extend_from_slice(format!("/P <> BDC\n").as_bytes()); + } + data.extend(b"0 0 1 1 re f\n".repeat(depth)); + data.extend(b"EMC\n".repeat(depth)); + data +} + +#[test] +fn comparing_deeply_nested_marked_content_holds_each_operator_once() { + let doc = one_page(b"", "", vec![]); + let data = nested_marked_content(2000); + let held = tags::retained_for_comparison(&doc, &Dict::new(), &data); + assert!(held <= 2 * data.len(), "{held} bytes held for {} bytes of content", data.len()); +} + +#[test] +fn a_change_inside_nested_marked_content_touches_every_enclosing_mcid() { + let doc = one_page(b"", "", vec![]); + let before = b"/P <> BDC /P <> BDC BT (A) Tj ET EMC 0 0 1 1 re f EMC /P <> BDC BT (B) Tj ET EMC"; + let after = b"/P <> BDC /P <> BDC BT ET EMC 0 0 1 1 re f EMC /P <> BDC BT (B) Tj ET EMC"; + let (changed, empty) = tags::touched(&doc, &Dict::new(), before, after); + assert_eq!(changed, HashSet::from([0, 1])); + assert_eq!(empty, HashSet::from([1])); +} diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs new file mode 100644 index 0000000000..d3aa98af67 --- /dev/null +++ b/crates/redact/src/tests/surviving_references.rs @@ -0,0 +1,68 @@ +//! Redacted content must leave the file: no other reference may keep it reachable by a full save. + +use super::*; + +/// Whether `secret` survives a full save of `doc`, in the file's bytes or in a decoded stream. +pub(super) fn full_save_holds(doc: &Document, secret: &[u8]) -> bool { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let found = |data: &[u8]| data.windows(secret.len()).any(|w| w == secret); + if found(&bytes) { + return true; + } + let saved = Document::open(Arc::new(bytes)).unwrap(); + saved.scan_objects().into_iter().any(|(_, o)| matches!(&*o, Object::Stream(s) if s.decoded().is_ok_and(|d| found(&d)))) +} + +#[test] +fn a_redacted_field_listed_in_the_calculation_order_leaves_the_saved_file() { + let mut doc = one_page(b"", "", vec![]); + pdfcraft_forms::add_field(&mut doc, 0, [10.0, 50.0, 100.0, 70.0], &pdfcraft_forms::NewField::Text { multiline: false }, Some("ssn")).unwrap(); + pdfcraft_forms::set_value(&mut doc, "ssn", &pdfcraft_forms::FieldValue::Text("SSN-078-05-1120".into())).unwrap(); + let field = pdfcraft_forms::fields(&doc).swap_remove(0).obj; + let root = doc.root().unwrap(); + let af = doc.get(root).as_dict().unwrap().reference(b"AcroForm").unwrap(); + doc.update_dict(af, |d| d.set(b"CO".to_vec(), Object::Array(vec![Object::Ref(field)]))).unwrap(); + mark(&mut doc, 0, &[[0.0, 40.0, 120.0, 80.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().fields, 1); + assert!(!full_save_holds(&doc, b"SSN-078-05-1120"), "the field's value is still reachable"); +} + +/// A surviving form shares the page's resource dictionary, which lists the image the mark +/// removes; the form never draws it. +#[test] +fn a_redacted_image_does_not_survive_in_a_surviving_forms_resources() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 7 0 R >>".to_vec(), + stream("", b"q 100 0 0 100 0 0 cm /Im Do Q /Fm Do"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Font << /F1 5 0 R >> /XObject << /Im 8 0 R /Fm 9 0 R >> >>".to_vec(), + stream("/Type /XObject /Subtype /Image /Width 11 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"IMAGESECRET"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources 7 0 R", b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]); + mark(&mut doc, 0, &[[0.0, 0.0, 120.0, 120.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_removed, 1); + assert!(content(&doc, 0).contains("Do"), "the form is still drawn"); + assert!(!full_save_holds(&doc, b"IMAGESECRET"), "the removed image is still reachable"); +} + +/// The same form drawing the image itself keeps it: the page still shows it there. +#[test] +fn an_image_a_surviving_form_draws_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 7 0 R >>".to_vec(), + stream("", b"q 100 0 0 100 0 0 cm /Im Do Q /Fm Do"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Font << /F1 5 0 R >> /XObject << /Im 8 0 R /Fm 9 0 R >> >>".to_vec(), + stream("/Type /XObject /Subtype /Image /Width 11 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"IMAGESECRET"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources 7 0 R", b"q 50 0 0 50 200 200 cm /Im Do Q"), + ]); + mark(&mut doc, 0, &[[0.0, 0.0, 120.0, 120.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_removed, 1); + assert!(full_save_holds(&doc, b"IMAGESECRET"), "the image the form still draws outside the mark"); +} diff --git a/crates/redact/src/tests/text_state.rs b/crates/redact/src/tests/text_state.rs new file mode 100644 index 0000000000..4e888f262c --- /dev/null +++ b/crates/redact/src/tests/text_state.rs @@ -0,0 +1,27 @@ +//! Glyphs are placed with the whole text state, wherever it was set. + +use super::*; + +const SECRET_AT_10_100: [f64; 4] = [5.0, 95.0, 45.0, 110.0]; + +#[test] +fn a_font_set_by_an_extgstate_places_its_glyphs() { + let mut doc = one_page(b"BT /GS1 gs 10 100 Td (SECRET) Tj ET", "/ExtGState << /GS1 << /Type /ExtGState /Font [5 0 R 10] >> >>", vec![]); + // "ET" spans x 30–40. + assert_eq!(under(&mut doc, 0, &[[30.0, 95.0, 40.0, 110.0]]), 2); + mark(&mut doc, 0, &[SECRET_AT_10_100], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("SECRET")); +} + +#[test] +fn a_form_inherits_the_text_state_of_the_page_that_draws_it() { + // The form shows text with the font, size and spacing the page set before `Do`. + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"BT 10 100 Td (SECRET) Tj ET"); + let mut doc = one_page(b"/F1 10 Tf 1 Tc /Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + // Glyphs 5 pt wide, 1 pt apart: the last one spans x 40–45. + assert_eq!(under(&mut doc, 0, &[[40.0, 95.0, 46.0, 110.0]]), 1); + mark(&mut doc, 0, &[SECRET_AT_10_100], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.glyphs, r.forms_rewritten), (6, 1), "{r:?}"); +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 5cdbf7d7a6..95991e7c8b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5709,8 +5709,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/ui-egui/tests/it/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/it/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::redaction_is_refused_when_the_page_still_shows_text_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources"] -notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text whose font is set through an ExtGState or taken by a form from its caller, some predefined CMaps and rotated Type 3 fonts are not removed yet, so those documents cannot be redacted. Removed images and forms are unlinked from page-tree resources (own, inherited or shared), not yet from a form's own resources, annotation appearances or patterns" +evidence = ["crates/ui-egui/tests/it/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/it/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::the_renderer_check_refuses_text_still_shown_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources"] +notes = "confirmation; one undo step; next save is a full rewrite. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" [[feature]] id = "protect.redact-glyph-removal"