M8: Bound and reuse repeated redaction overlays (#1426)

Consolidate the reviewed change and its follow-ups on current upstream.

Original change notes:
M8: Bound and reuse repeated redaction overlays

Prepare appearances within a shared 8 MiB budget before changing the document. Reuse one measured and encoded row, reject excessive geometry, and preserve document and undo state on refusal.

Fixes #1326

M8: Preserve upstream font metrics in bounded redaction overlays

M8: reuse ordered redaction mark groups (#1326)

Co-authored-by: LabDennis <341377186+LabDennis@users.noreply.github.com>
This commit is contained in:
Sir Clangington III.
2026-10-15 02:16:56 -04:00
committed by GitHub
co-authored by LabDennis
parent fc601b89e1
commit 8a04b6d5c2
5 changed files with 315 additions and 80 deletions
+25
View File
@@ -1802,6 +1802,31 @@ fn redaction_marks_apply_for_good_and_undo() {
assert!(matches!(s.apply(id, Edit::ApplyRedactions { pages: None }), Err(EditError::Redact(_))));
}
#[test]
fn oversized_redaction_overlay_preserves_the_document_and_undo() {
let (mut s, id) = session_with(1);
let shape = Shape::Redact {
quads: vec![pdfcraft_annot::rect_quad([0.0, 0.0, 16000.0, 1600.0])],
overlay: "X".into(),
look: pdfcraft_annot::OverlayLook { size: 2.0, repeat: true, ..Default::default() },
};
s.apply(
id,
Edit::AddAnnotation(NewAnnotation { page: 0, style: Style::default_for(&shape), shape, contents: String::new(), author: "Tester".into() }),
)
.unwrap();
let before = s.get(id).unwrap().bytes.clone();
assert!(matches!(s.apply(id, Edit::ApplyRedactions { pages: None }), Err(EditError::Redact(pdfcraft_redact::RedactError::OverlayLimit))));
let doc = s.get(id).unwrap();
assert_eq!(doc.bytes, before);
assert_eq!(doc.redaction_marks(), 1);
assert_eq!(doc.can_undo(), Some("Add redaction mark"));
assert_eq!(page_texts(&s, id), ["Page 1"]);
s.undo(id).unwrap();
assert_eq!(s.get(id).unwrap().redaction_marks(), 0);
assert_eq!(page_texts(&s, id), ["Page 1"]);
}
#[test]
fn printing_lays_out_sheets_that_render() {
let (s, id) = session_with(5);
+7
View File
@@ -34,6 +34,13 @@ A verification pass re-reads each redacted page and fails the operation if any g
image is still under an area. Content that can't be decoded makes the operation fail rather than
leave it unredacted.
Before changing any page, `apply` prepares the redaction appearances with a shared 8 MiB
budget for their uncompressed PDF content across the selected pages. It bounds repeated-text
counts and working line buffers before allocating them, and measures and encodes each repeated
line once. Invalid or excessive appearances return `RedactError::OverlayLimit` without changing
the document. Reduce the marked area, disable repeated text, or apply fewer pages at a time.
This limits appearance generation; it is not a limit on the input PDF or the whole process's memory.
Structure cleanup resolves named MCIDs and indirect content references. Form content
references are updated to the cleaned copy; unchanged sibling descriptions remain.
+19 -80
View File
@@ -21,6 +21,7 @@ use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream};
pub mod codes;
mod image;
mod interp;
mod overlay;
pub mod patterns;
pub mod sanitize;
mod tags;
@@ -46,6 +47,10 @@ pub enum RedactError {
"page {0} has vertical-writing text while a redaction mark is pending, and PdfCraft cannot tell whether that text lies under the mark yet, so nothing was changed"
)]
VerticalText(usize),
#[error(
"redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)"
)]
OverlayLimit,
#[error(transparent)]
Edit(#[from] pdfcraft_edit::EditError),
#[error(transparent)]
@@ -206,76 +211,6 @@ pub(crate) fn page_streams(doc: &Document, page: &Dict, index: usize) -> Result<
Ok((list, data))
}
/// The boxes and overlay text drawn for the applied marks.
fn overlay_content(marks: &[&Mark]) -> Vec<u8> {
let n = |v: f64| {
let s = format!("{:.3}", v);
s.trim_end_matches('0').trim_end_matches('.').to_string()
};
let mut c: Vec<u8> = Vec::new();
for m in marks {
let Some(fill) = m.fill else { continue };
c.extend(format!("q {} {} {} rg\n", n(fill[0]), n(fill[1]), n(fill[2])).bytes());
for r in &m.rects {
c.extend(format!("{} {} {} {} re f\n", n(r[0]), n(r[1]), n(r[2] - r[0]), n(r[3] - r[1])).bytes());
}
c.extend_from_slice(b"Q\n");
if m.overlay.is_empty() {
continue;
}
// The overlay text: its font, colour and alignment; auto-sized to fit unless a size is
// set; repeated to fill the area when asked.
let look = &m.look;
// The published standard-14 metrics, so overlay text is measured as it will be drawn.
let (res, face) = match look.font {
pdfcraft_annot::OverlayFont::Helvetica => ("PCHelv", pdfcraft_fonts::Std14::Helvetica),
pdfcraft_annot::OverlayFont::Times => ("PCTimes", pdfcraft_fonts::Std14::TimesRoman),
pdfcraft_annot::OverlayFont::Courier => ("PCCour", pdfcraft_fonts::Std14::Courier),
};
let width = move |s: &str, size: f64| face.text_width(s, size);
let [cr, cg, cb] = look.color.map(|v| v.clamp(0.0, 1.0));
for r in &m.rects {
let (w, h) = (r[2] - r[0], r[3] - r[1]);
let mut size = if look.size > 0.0 { look.size } else { (h * 0.7).min(12.0) };
let tw = width(&m.overlay, size);
if look.size <= 0.0 && tw > w - 2.0 && tw > 0.0 {
size *= (w - 2.0).max(0.0) / tw;
}
if size < 2.0 {
continue;
}
// One line, or as many repeated lines as fit (each line the text repeated across).
let lines: Vec<String> = if look.repeat {
let unit = width(&format!("{} ", m.overlay), size).max(0.01);
let per_line = ((w - 2.0) / unit).floor().max(1.0) as usize;
let count = ((h / (size * 1.2)).floor() as usize).max(1);
vec![vec![m.overlay.as_str(); per_line].join(" "); count]
} else {
vec![m.overlay.clone()]
};
let block = lines.len() as f64 * size * 1.2;
let mut y = r[1] + (h + block) / 2.0 - size * 0.95;
c.extend(
format!("q {} {} {} {} re W n BT {} {} {} rg /{res} {} Tf ", n(r[0]), n(r[1]), n(w), n(h), n(cr), n(cg), n(cb), n(size)).bytes(),
);
for line in &lines {
let lw = width(line, size);
let x = match look.align {
0 => r[0] + 1.0,
2 => r[2] - 1.0 - lw,
_ => r[0] + (w - lw) / 2.0,
};
c.extend(format!("1 0 0 1 {} {} Tm ", n(x), n(y)).bytes());
c.extend_from_slice(&pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(line)));
c.extend_from_slice(b" Tj ");
y -= size * 1.2;
}
c.extend_from_slice(b"ET Q\n");
}
}
c
}
/// The nearest ancestor of `page` that holds `/Resources` (where an inheriting page gets them):
/// `Some(None)` when none does, `None` when the chain can't be followed (a cycle, a broken node or
/// a tree deeper than any real one), so callers can stay conservative.
@@ -384,9 +319,16 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Reda
)));
}
let mut report = Report { marks: chosen.len(), ..Report::default() };
let mut by_page: Vec<usize> = chosen.iter().map(|m| m.page).collect();
by_page.sort_unstable();
by_page.dedup();
// `marks` visits pages in order. Keep slices of that order for appearance generation,
// content rewriting and verification instead of scanning all marks for every page.
let groups: Vec<_> = chosen.chunk_by(|a, b| a.page == b.page).filter_map(|group| group.first().map(|mark| (mark.page, group))).collect();
let by_page: Vec<_> = groups.iter().map(|(page, _)| *page).collect();
// Budget failures must leave even a direct library caller's document untouched.
let mut remaining = overlay::MAX_BYTES;
let overlays = groups
.iter()
.map(|&(pi, page_marks)| overlay::content(page_marks, &mut remaining).map(|content| (pi, content)))
.collect::<Result<Vec<_>, _>>()?;
report.pages = by_page.len();
localize_resources(doc, &by_page)?;
let mut doomed_fields: Vec<String> = Vec::new();
@@ -395,9 +337,8 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Reda
let widget_owner: Vec<(ObjRef, String)> =
pdfcraft_forms::fields(doc).into_iter().flat_map(|f| f.widgets.iter().map(|w| (w.obj, f.name.clone())).collect::<Vec<_>>()).collect();
for &pi in &by_page {
for &(pi, page_marks) in &groups {
let page = pdfcraft_model::pages(doc).swap_remove(pi);
let page_marks: Vec<&Mark> = chosen.iter().copied().filter(|m| m.page == pi).collect();
let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect();
// 1. Content.
@@ -558,9 +499,7 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Reda
}
// 4. The boxes.
for &pi in &by_page {
let page_marks: Vec<&Mark> = chosen.iter().copied().filter(|m| m.page == pi).collect();
let content = overlay_content(&page_marks);
for (pi, content) in overlays {
if !content.is_empty() {
pdfcraft_edit::stamp(doc, pi, "Redaction", content)?;
}
@@ -572,9 +511,9 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Reda
// 5. Verify: nothing readable may remain under a region.
let mut residue = 0;
for &pi in &by_page {
for &(pi, page_marks) in &groups {
let page = pdfcraft_model::pages(doc).swap_remove(pi);
let rects: Vec<[f64; 4]> = chosen.iter().filter(|m| m.page == pi).flat_map(|m| m.rects.iter().copied()).collect();
let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect();
let (list, data) = page_streams(doc, &page.dict, pi)?;
// The overlay stream (last) draws text of its own: the overlay label is allowed.
let original: Vec<Vec<u8>> = list
+129
View File
@@ -0,0 +1,129 @@
//! Prepare bounded redaction appearances before the document is changed.
use std::borrow::Cow;
use super::{Mark, RedactError};
/// Shared by every selected page, including fills, text and PDF operators.
pub(super) const MAX_BYTES: usize = 8 * 1024 * 1024;
struct Output<'a> {
bytes: Vec<u8>,
remaining: &'a mut usize,
}
impl Output<'_> {
fn push(&mut self, bytes: impl AsRef<[u8]>) -> Result<(), RedactError> {
let bytes = bytes.as_ref();
let left = self.remaining.checked_sub(bytes.len()).ok_or(RedactError::OverlayLimit)?;
self.bytes.try_reserve(bytes.len()).map_err(|_| RedactError::OverlayLimit)?;
self.bytes.extend_from_slice(bytes);
*self.remaining = left;
Ok(())
}
}
fn repeat_line(text: &str, across: f64, down: f64, remaining: usize) -> Result<(String, usize), RedactError> {
// Check floating-point counts before casting, and the product before allocating.
if !across.is_finite() || !down.is_finite() || across > remaining as f64 || down > remaining as f64 {
return Err(RedactError::OverlayLimit);
}
let (across, down) = (across as usize, down as usize);
let bytes = text.len().checked_add(1).and_then(|n| n.checked_mul(across)).and_then(|n| n.checked_sub(1));
let bytes = bytes.filter(|n| *n <= remaining).ok_or(RedactError::OverlayLimit)?;
let mut line = String::new();
line.try_reserve_exact(bytes).map_err(|_| RedactError::OverlayLimit)?;
for i in 0..across {
if i > 0 {
line.push(' ');
}
line.push_str(text);
}
Ok((line, down))
}
pub(super) fn content(marks: &[&Mark], remaining: &mut usize) -> Result<Vec<u8>, RedactError> {
let n = |v: f64| {
let s = format!("{:.3}", v);
s.trim_end_matches('0').trim_end_matches('.').to_string()
};
let mut c = Output { bytes: Vec::new(), remaining };
for m in marks {
let Some(fill) = m.fill else { continue };
if !fill.iter().all(|v| v.is_finite()) || m.rects.iter().flatten().any(|v| !v.is_finite()) {
return Err(RedactError::OverlayLimit);
}
c.push(format!("q {} {} {} rg\n", n(fill[0]), n(fill[1]), n(fill[2])))?;
for r in &m.rects {
let (w, h) = (r[2] - r[0], r[3] - r[1]);
if !w.is_finite() || !h.is_finite() {
return Err(RedactError::OverlayLimit);
}
c.push(format!("{} {} {} {} re f\n", n(r[0]), n(r[1]), n(w), n(h)))?;
}
c.push(b"Q\n")?;
if m.overlay.is_empty() {
continue;
}
if m.overlay.len() > *c.remaining {
return Err(RedactError::OverlayLimit);
}
let look = &m.look;
if !look.size.is_finite() || !look.color.iter().all(|v| v.is_finite()) {
return Err(RedactError::OverlayLimit);
}
// Keep upstream's standard-14 metrics when preparing the bounded appearance.
let (res, face) = match look.font {
pdfcraft_annot::OverlayFont::Helvetica => ("PCHelv", pdfcraft_fonts::Std14::Helvetica),
pdfcraft_annot::OverlayFont::Times => ("PCTimes", pdfcraft_fonts::Std14::TimesRoman),
pdfcraft_annot::OverlayFont::Courier => ("PCCour", pdfcraft_fonts::Std14::Courier),
};
let width = move |s: &str, size: f64| face.text_width(s, size);
let [cr, cg, cb] = look.color.map(|v| v.clamp(0.0, 1.0));
for r in &m.rects {
let (w, h) = (r[2] - r[0], r[3] - r[1]);
let mut size = if look.size > 0.0 { look.size } else { (h * 0.7).min(12.0) };
let tw = width(&m.overlay, size);
if look.size <= 0.0 && tw > w - 2.0 && tw > 0.0 {
size *= (w - 2.0).max(0.0) / tw;
}
if size < 2.0 {
continue;
}
let (line, count) = if look.repeat {
let unit = width(&format!("{} ", m.overlay), size).max(0.01);
let across = ((w - 2.0) / unit).floor().max(1.0);
let down = (h / (size * 1.2)).floor().max(1.0);
let (line, count) = repeat_line(&m.overlay, across, down, *c.remaining)?;
(Cow::Owned(line), count)
} else {
(Cow::Borrowed(m.overlay.as_str()), 1)
};
// Every row is identical: measure and encode it once, without a Vec of copies.
let encoded = pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&line));
if encoded.len().checked_add(4).and_then(|n| n.checked_mul(count)).is_none_or(|n| n > *c.remaining) {
return Err(RedactError::OverlayLimit);
}
let lw = width(&line, size);
let x = match look.align {
0 => r[0] + 1.0,
2 => r[2] - 1.0 - lw,
_ => r[0] + (w - lw) / 2.0,
};
let block = count as f64 * size * 1.2;
let mut y = r[1] + (h + block) / 2.0 - size * 0.95;
if !lw.is_finite() || !x.is_finite() || !y.is_finite() {
return Err(RedactError::OverlayLimit);
}
c.push(format!("q {} {} {} {} re W n BT {} {} {} rg /{res} {} Tf ", n(r[0]), n(r[1]), n(w), n(h), n(cr), n(cg), n(cb), n(size)))?;
for _ in 0..count {
c.push(format!("1 0 0 1 {} {} Tm ", n(x), n(y)))?;
c.push(&encoded)?;
c.push(b" Tj ")?;
y -= size * 1.2;
}
c.push(b"ET Q\n")?;
}
}
Ok(c.bytes)
}
+135
View File
@@ -685,6 +685,141 @@ fn overlay_text_takes_its_font_size_colour_alignment_and_repeats() {
assert!(fonts.contains(b"PCCour") && !fonts.contains(b"PCTimes"));
}
fn repeated_overlay_document(width: f64, height: f64, repeat: bool) -> Document {
pdf(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Annots [5 0 R] >>".to_vec(),
stream("", b"0 0 20 20 re f"),
format!("<< /Type /Annot /Subtype /Redact /Rect [0 0 {width} {height}] /IC [0 0 0] /OverlayText (X) /DA (/Helv 2 Tf) /Repeat {repeat} >>")
.into_bytes(),
])
}
#[test]
fn oversized_repeated_overlay_is_refused_before_changing_the_document() {
// About 11 MiB of generated content before the fix; a bounded reproduction, not an OOM test.
let mut doc = repeated_overlay_document(16000.0, 1600.0, true);
let before = write_full(&doc, &SaveOptions::default()).unwrap();
assert_eq!(apply(&mut doc, None), Err(RedactError::OverlayLimit));
assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before);
assert_eq!(marks(&doc).len(), 1);
let mut single = repeated_overlay_document(16000.0, 1600.0, false);
apply(&mut single, None).unwrap();
assert_eq!(content(&reopen(&single), 0).matches("(X) Tj").count(), 1);
}
#[test]
fn overlay_budget_is_shared_by_pages_and_respects_the_page_selection() {
let mut doc = repeated_overlay_document(8000.0, 800.0, true);
let page = (*doc.get(ObjRef::new(3, 0))).clone();
let mut kids = vec![Object::Ref(ObjRef::new(3, 0))];
for _ in 0..2 {
let mut page = page.as_dict().unwrap().clone();
let annotation = doc.add((*doc.get(ObjRef::new(5, 0))).clone());
page.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]));
kids.push(Object::Ref(doc.add(Object::Dict(page))));
}
doc.update_dict(ObjRef::new(2, 0), |d| {
d.set(b"Kids".to_vec(), Object::Array(kids));
d.set(b"Count".to_vec(), Object::Int(3));
})
.unwrap();
let before = write_full(&doc, &SaveOptions::default()).unwrap();
// Each appearance fits on its own; their sum exceeds the operation's budget.
assert_eq!(apply(&mut doc, None), Err(RedactError::OverlayLimit));
assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before);
assert_eq!(marks(&doc).len(), 3);
apply(&mut doc, Some(&[1])).unwrap();
assert_eq!(marks(&doc).iter().map(|m| m.page).collect::<Vec<_>>(), [0, 2]);
}
#[test]
fn grouped_marks_keep_overlay_order_and_ignore_duplicate_page_selections() {
let mut doc = repeated_overlay_document(60.0, 30.0, false);
let page = doc.get(ObjRef::new(3, 0)).as_dict().unwrap().clone();
let original_mark = (*doc.get(ObjRef::new(5, 0))).clone();
let mut kids = vec![Object::Ref(ObjRef::new(3, 0))];
for _ in 0..2 {
let mut copy = page.clone();
let mark = doc.add(original_mark.clone());
copy.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(mark)]));
kids.push(Object::Ref(doc.add(Object::Dict(copy))));
}
doc.update_dict(ObjRef::new(2, 0), |d| {
d.set(b"Kids".to_vec(), Object::Array(kids));
d.set(b"Count".to_vec(), Object::Int(3));
})
.unwrap();
let mut second = original_mark.as_dict().unwrap().clone();
second.set(b"OverlayText".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"Y".to_vec())));
let second = doc.add(Object::Dict(second));
doc.update_dict(ObjRef::new(3, 0), |d| {
d.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(5, 0)), Object::Ref(second)]));
})
.unwrap();
let untouched = content(&doc, 1);
let report = apply(&mut doc, Some(&[2, 0, 2])).unwrap();
assert_eq!((report.pages, report.marks), (2, 3));
let saved = reopen(&doc);
assert_eq!(marks(&saved).iter().map(|m| m.page).collect::<Vec<_>>(), [1]);
assert_eq!(content(&saved, 1), untouched);
let first = content(&saved, 0);
assert!(first.find("(X) Tj").unwrap() < first.find("(Y) Tj").unwrap());
assert_eq!(content(&saved, 2).matches("(X) Tj").count(), 1);
}
#[test]
fn overlay_extreme_counts_and_invalid_geometry_are_refused_without_large_allocations() {
let doc = repeated_overlay_document(20.0, 10.0, true);
let original = marks(&doc).remove(0);
for rect in [[0.0, 0.0, 1e100, 10.0], [0.0, 0.0, 20.0, 1e100], [-f64::MAX, 0.0, f64::MAX, 10.0], [0.0, 0.0, f64::NAN, 10.0]] {
let mut mark = original.clone();
mark.rects = vec![rect];
let mut remaining = overlay::MAX_BYTES;
assert_eq!(overlay::content(&[&mark], &mut remaining), Err(RedactError::OverlayLimit));
}
}
#[test]
fn repeated_overlay_keeps_its_exact_appearance_and_counts_escaped_bytes() {
let doc = repeated_overlay_document(20.0, 10.0, true);
let mut mark = marks(&doc).remove(0);
let expected = b"q 0 0 0 rg\n0 0 20 10 re f\nQ\nq 0 0 20 10 re W n BT 1 0 0 rg /PCHelv 2 Tf 1 0 0 1 1.773 7.9 Tm (X X X X X X X X X) Tj 1 0 0 1 1.773 5.5 Tm (X X X X X X X X X) Tj 1 0 0 1 1.773 3.1 Tm (X X X X X X X X X) Tj 1 0 0 1 1.773 0.7 Tm (X X X X X X X X X) Tj ET Q\n";
let mut remaining = expected.len();
assert_eq!(overlay::content(&[&mark], &mut remaining).unwrap(), expected);
assert_eq!(remaining, 0);
mark.overlay = "é(\\)".into();
let mut remaining = overlay::MAX_BYTES;
let encoded = overlay::content(&[&mark], &mut remaining).unwrap();
let escaped = b"\xe9\\(\\\\\\)";
assert!(encoded.windows(escaped.len()).any(|w| w == escaped));
assert_eq!(overlay::MAX_BYTES - remaining, encoded.len());
let mut too_small = encoded.len() - 1;
assert_eq!(overlay::content(&[&mark], &mut too_small), Err(RedactError::OverlayLimit));
}
#[test]
#[ignore = "manual overlay timing probe"]
fn repeated_overlay_timing_probe() {
use std::hash::{Hash, Hasher};
for (w, h) in [(20.0, 10.0), (2000.0, 200.0), (4000.0, 400.0), (8000.0, 800.0)] {
let doc = repeated_overlay_document(w, h, true);
let marks = marks(&doc);
let refs: Vec<_> = marks.iter().collect();
let start = std::time::Instant::now();
let mut bytes = Vec::new();
for _ in 0..20 {
let mut remaining = overlay::MAX_BYTES;
bytes = std::hint::black_box(overlay::content(std::hint::black_box(&refs), &mut remaining).unwrap());
}
let elapsed = start.elapsed().as_secs_f64() * 1000.0 / 20.0;
let mut hash = std::collections::hash_map::DefaultHasher::new();
bytes.hash(&mut hash);
println!("{w}x{h}: bytes={} hash={} ms={elapsed:.3}", bytes.len(), hash.finish());
}
}
/// Overlay text the standard fonts can't draw: refused when it is marked, and for a mark another
/// program made, refused when it is applied, before anything is redacted, rather than burnt into
/// the page as `?`.