diff --git a/crates/audit/src/lib.rs b/crates/audit/src/lib.rs index f1f1350987..f2bcf63358 100644 --- a/crates/audit/src/lib.rs +++ b/crates/audit/src/lib.rs @@ -138,16 +138,25 @@ struct Budget { } impl Budget { - /// Resets the comparison count for a page with these `lines`. + /// Resets the comparison count for a page with these `lines`. A line costs its word boxes + /// or its runs, whichever are more (at least one): a cover check walks the boxes, and the + /// covered text walks the runs, which can draw no words at all (`( ) Tj`). fn start_page(&mut self, lines: &[TextLine]) { self.comparisons = 0; - self.page_cost = lines.iter().fold(0, |n, line| n.saturating_add(line.ink.len().max(1))); + self.page_cost = lines.iter().fold(0, |n, line| n.saturating_add(line.ink.len().max(line.runs().count()).max(1))); } - /// The text of the `lines` that `hides`, joined with single spaces, or `None` when they show - /// no text. Without `copying` the text is empty and never assembled: one long line under many - /// covers would otherwise be copied once per cover, outside both bounds. - fn hidden_text(&mut self, lines: &[TextLine], copying: bool, hides: impl Fn(&TextLine) -> bool) -> Result, AuditError> { + /// The text `cover` hides in the `lines` that `hides` ([`covered_words`]), joined with single + /// spaces, or `None` when they show no text. Without `copying` the text is empty and never + /// assembled: one long line under many covers would otherwise be copied once per cover, + /// outside both bounds. + fn hidden_text( + &mut self, + lines: &[TextLine], + copying: bool, + cover: &[f64; 4], + hides: impl Fn(&TextLine) -> bool, + ) -> Result, AuditError> { self.comparisons = self.comparisons.saturating_add(self.page_cost); if self.comparisons > self.limits.comparisons { return Err(AuditError::TooManyComparisons); @@ -160,7 +169,7 @@ impl Budget { if !covered.is_empty() { covered.push(' '); } - covered.push_str(line.text.trim()); + covered_words(cover, line, &mut covered); } Ok((!covered.trim().is_empty()).then_some(covered)) } @@ -188,14 +197,17 @@ fn audit(doc: &Document, limits: Limits) -> Result, AuditError budget.start_page(&lines); // 1. Opaque filled rectangles over extractable text. for r in cover::filled_rects(doc, &page.dict)? { - if let Some(covered) = budget.hidden_text(&lines, copying, |line| is_cover(&r, line))? { + if let Some(covered) = budget.hidden_text(&lines, copying, &r.rect, |line| is_cover(&r, line))? { let covered_text = budget.report_text(covered, copying)?; out.push(AuditFinding { page: n, kind: FindingKind::CoveredText, rect: r.rect, covered_text }); } } // 2. Annotation covers over extractable text. for annot in annots(doc, &page.dict)? { - if let Some(covered) = budget.hidden_text(&lines, copying, |line| match &annot { + let cover = match &annot { + AnnotationCover::Redact(rect) | AnnotationCover::Square { rect, .. } => *rect, + }; + if let Some(covered) = budget.hidden_text(&lines, copying, &cover, |line| match &annot { AnnotationCover::Redact(rect) => covers(rect, line), AnnotationCover::Square { rect, rgb } => covers(rect, line) && contrast(line.color, *rgb) < COVER_CONTRAST, })? { @@ -250,6 +262,34 @@ fn covers(cover: &[f64; 4], line: &TextLine) -> bool { inked || (boxed && (line.ink.is_empty() || word_intersects())) } +/// Appends the text of `line` that `cover` lies on to `out`: the runs whose words it overlaps +/// (#4164), so the words of the line painted beside the cover, which stay visible, are not +/// reported as hidden. Neighbouring runs are joined as the line joins them, so a word split +/// across runs stays one word; runs with others left out between them are spaced. The whole +/// line when its runs can't say: a single run, a line whose text was reordered for reading, or +/// a cover that overlaps the line's box but none of its words. +fn covered_words(cover: &[f64; 4], line: &TextLine, out: &mut String) { + let overlaps = |w: &[f64; 4]| area(&[cover[0].max(w[0]), cover[1].max(w[1]), cover[2].min(w[2]), cover[3].min(w[3])]) > 0.0; + let mut words = String::new(); + let mut last = None; + for (i, (text, ink, spaced)) in line.runs().enumerate() { + if text.trim().is_empty() || !ink.iter().any(overlaps) { + continue; + } + let apart = last.is_some_and(|l: usize| l.saturating_add(1) != i || spaced); + if apart && !words.ends_with(char::is_whitespace) && !text.starts_with(char::is_whitespace) { + words.push(' '); + } + words.push_str(text); + last = Some(i); + } + if line.runs().nth(1).is_none() || last.is_none() { + out.push_str(line.text.trim()); + } else { + out.push_str(words.trim()); + } +} + /// The area of a rectangle, 0 when it is empty or inverted. fn area(r: &[f64; 4]) -> f64 { (r[2] - r[0]).max(0.0) * (r[3] - r[1]).max(0.0) diff --git a/crates/audit/src/tests.rs b/crates/audit/src/tests.rs index 3aca406e7c..e5fb9ab683 100644 --- a/crates/audit/src/tests.rs +++ b/crates/audit/src/tests.rs @@ -645,6 +645,33 @@ fn later_text_on_the_line_does_not_clear_an_earlier_cover() { assert!(findings[0].covered_text.contains("SECRET"), "{findings:?}"); } +/// #4164: a cover over part of a line reports the words under it, not the words of the same +/// line painted beside it, which stay visible. +#[test] +fn covered_text_leaves_out_the_words_beside_the_cover() { + let covered = |content: &[u8]| { + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + findings.into_iter().map(|f| f.covered_text).collect::>() + }; + assert_eq!(covered(b"0 g\nBT /F1 24 Tf 100 700 Td (SECRET) Tj ET\n98 688 94 32 re f\nBT /F1 24 Tf 200 700 Td (tail) Tj ET\n"), ["SECRET"]); + // A word shown in two runs with nothing between them stays one word (review on #4198). + assert_eq!(covered(b"0 g\nBT /F1 24 Tf 100 700 Td (SE) Tj (CRET) Tj ET\n98 688 94 32 re f\nBT /F1 24 Tf 200 700 Td (tail) Tj ET\n"), ["SECRET"]); + + // The same for an unapplied redaction mark over the first run only. A mark over both runs + // reports both, with the space the line has between them (`tail` is set apart here). + for (rect, expected) in [("[98 688 192 720]", "SECRET"), ("[98 688 310 720]", "SECRET tail")] { + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>"); + b.add_stream(b"BT /F1 24 Tf 100 700 Td (SECRET) Tj ET\nBT /F1 24 Tf 250 700 Td (tail) Tj ET\n"); + b.add(FONT); + b.add(&format!("<< /Type /Annot /Subtype /Redact /Rect {rect} >>")); + let findings = audit_redactions(&open(b.build())).expect("audit must run"); + assert_eq!(findings.iter().map(|f| (f.kind.clone(), f.covered_text.as_str())).collect::>(), [(FindingKind::UnappliedMark, expected)]); + } +} + /// Many covers over the same long line repeat its text in every finding: a small file must not /// expand into an unbounded report. Past the budget the audit fails rather than truncate. #[test] @@ -677,6 +704,28 @@ fn cover_and_line_comparisons_are_bounded() { assert!(crate::audit(&doc, limits).expect("within the budget").is_empty()); } +/// A run that draws no word still costs a comparison: the covered text walks every run of a +/// covered line, so a word followed by many `( ) Tj` must not be checked for free (review on +/// #4198). +#[test] +fn runs_without_words_count_against_the_comparison_bound() { + let mut content = b"BT /F1 24 Tf 100 700 Td (SECRET) Tj".to_vec(); + content.extend_from_slice(&b" ( ) Tj".repeat(50)); + content.extend_from_slice( + b" ET +0 g +98 688 94 32 re f +", + ); + let doc = open(one_page(&content, "")); + let lines = pdfcraft_edit::glyph_text_lines(&doc, &pdfcraft_model::pages(&doc)[0]).expect("text"); + assert_eq!(lines.iter().map(|l| (l.ink.len(), l.runs().count())).collect::>(), [(1, 51)]); + let limits = crate::Limits { comparisons: 50, covered_text: usize::MAX }; + assert!(matches!(crate::audit(&doc, limits), Err(crate::AuditError::TooManyComparisons))); + let limits = crate::Limits { comparisons: 51, covered_text: usize::MAX }; + assert_eq!(crate::audit(&doc, limits).expect("within the budget").len(), 1); +} + /// Covered text is document text: a document whose permissions withhold copying reports where /// its covers are but not what they hide, unless it was opened with the owner password. #[test] @@ -714,14 +763,15 @@ fn copy_restricted_documents_withhold_covered_text() { /// per cover, a volume neither the comparison nor the output bound charges. #[test] fn withheld_covered_text_is_not_assembled() { + const PAGE: [f64; 4] = [0.0, 0.0, 612.0, 792.0]; let doc = open(one_page(b"BT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n", "")); let lines = pdfcraft_edit::text_lines(&doc, 0).expect("text"); let mut budget = crate::Budget { limits: crate::Limits::DEFAULT, comparisons: 0, page_cost: 0, covered_text: 0 }; budget.start_page(&lines); - let withheld = budget.hidden_text(&lines, false, |_| true).expect("within the budget"); + let withheld = budget.hidden_text(&lines, false, &PAGE, |_| true).expect("within the budget"); assert_eq!(withheld.map(|t| t.capacity()), Some(0), "a finding, with no text built for it"); - assert_eq!(budget.hidden_text(&lines, false, |_| false).expect("within the budget"), None); - let copied = budget.hidden_text(&lines, true, |_| true).expect("within the budget"); + assert_eq!(budget.hidden_text(&lines, false, &PAGE, |_| false).expect("within the budget"), None); + let copied = budget.hidden_text(&lines, true, &PAGE, |_| true).expect("within the budget"); assert_eq!(copied.as_deref(), Some("SECRET DATA 123")); } diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index ff16b63c82..424d5d28ac 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -80,6 +80,10 @@ pub struct TextLine { /// Why the line can't be edited: its font's writing mode (its CMap's `WMode`) couldn't be read, /// so which way its glyphs advance is unknown. The line is listed as horizontal text. unreadable_mode: Option, + /// The runs the line joins, in content order: each one's text, the end of its words in + /// `ink`, and whether joining put a space before it. Empty when the line's text was put + /// into another order (see [`TextLine::runs`]). + runs: Vec<(String, usize, bool)>, } impl TextLine { @@ -591,6 +595,20 @@ impl TextLine { (self.stream, self.ops.iter().min().copied().unwrap_or(0)) } + /// The runs of text the line joins, in content order: each run's text as its glyphs show it, + /// the boxes of its words (a part of `ink`), and whether joining put a space between it and + /// the run before (`text` has one there; runs closer together join with none, so one word + /// can span runs). None when the line's text was reordered for reading (right-to-left or + /// visual-order Indic text): then only the line's text as a whole says what it reads. + pub fn runs(&self) -> impl Iterator { + let mut start = 0; + self.runs.iter().map(move |(text, end, spaced)| { + let ink = self.ink.get(start..*end).unwrap_or_default(); + start = *end; + (text.as_str(), ink, *spaced) + }) + } + fn rotation(&self) -> f64 { let m = Matrix(self.origin.tm).then(&Matrix(self.origin.ctm)).0; m[1].atan2(m[0]).to_degrees() @@ -1485,13 +1503,15 @@ fn text_lines_in(doc: &Document, p: &pdfcraft_model::Page, actual_text: bool) -> }); if joins && let Some(l) = lines.last_mut() { let gap = last.map_or(0.0, |(_, pen)| pen.continues(&s, 0.0).1); - if gap - s.tracking > l.size * s.state.scale.abs() * 0.2 && !l.text.ends_with(' ') && !s.text.starts_with(' ') { + let spaced = gap - s.tracking > l.size * s.state.scale.abs() * 0.2 && !l.text.ends_with(' ') && !s.text.starts_with(' '); + if spaced { l.text.push(' '); } l.text.push_str(&s.text); l.rect = [l.rect[0].min(s.rect[0]), l.rect[1].min(s.rect[1]), l.rect[2].max(s.rect[2]), l.rect[3].max(s.rect[3])]; l.frame = union(l.frame, s.frame); l.ink.extend(s.ink.iter().copied()); + l.runs.push((s.text.clone(), l.ink.len(), spaced)); l.ops.push(s.op); l.ops.extend(s.extra_ops.iter().copied()); l.decodable &= s.decodable; @@ -1524,6 +1544,7 @@ fn text_lines_in(doc: &Document, p: &pdfcraft_model::Page, actual_text: bool) -> actual_text: s.actual_text, clips: s.state.render_mode >= 4, unreadable_mode: s.state.unreadable.as_deref().map(str::to_owned), + runs: vec![(s.text.clone(), s.ink.len(), false)], ops: std::iter::once(s.op).chain(s.extra_ops.iter().copied()).collect(), origin: Origin { tm: s.tm.0, @@ -1543,10 +1564,15 @@ fn text_lines_in(doc: &Document, p: &pdfcraft_model::Page, actual_text: bool) -> // Content streams draw right-to-left lines in visual order; the editor works in typing order. for l in lines.iter_mut().filter(|l| !l.logical_order) { // A visual-order Indic font draws pre-base vowel signs before their consonant. + let shown = l.text.clone(); if l.visual_order && !l.actual_text { l.text = crate::indic::to_logical(&l.text); } l.text = crate::bidi::to_logical(&l.text); + // The runs hold the text in drawing order, which no longer reads as the line does. + if l.text != shown { + l.runs.clear(); + } } // Lines of only spaces aren't editable text. lines.retain(|l| !l.text.trim().is_empty()); @@ -1592,13 +1618,15 @@ fn add_shown(lines: &mut Vec, last: &mut Option, s: Shown, stream }); if joins && let Some(l) = lines.last_mut() { let gap = last.map_or(0.0, |pen| pen.continues(&s, 0.0).1); - if gap - s.tracking > l.size * s.state.scale.abs() * 0.2 && !l.text.ends_with(' ') && !s.text.starts_with(' ') { + let spaced = gap - s.tracking > l.size * s.state.scale.abs() * 0.2 && !l.text.ends_with(' ') && !s.text.starts_with(' '); + if spaced { l.text.push(' '); } l.text.push_str(&s.text); l.rect = [l.rect[0].min(s.rect[0]), l.rect[1].min(s.rect[1]), l.rect[2].max(s.rect[2]), l.rect[3].max(s.rect[3])]; l.frame = union(l.frame, s.frame); l.ink.extend(s.ink.iter().copied()); + l.runs.push((s.text.clone(), l.ink.len(), spaced)); l.ops.push(s.op); l.ops.extend(s.extra_ops.iter().copied()); l.decodable &= s.decodable; @@ -1630,6 +1658,7 @@ fn add_shown(lines: &mut Vec, last: &mut Option, s: Shown, stream actual_text: s.actual_text, clips: s.state.render_mode >= 4, unreadable_mode: s.state.unreadable.as_deref().map(str::to_owned), + runs: vec![(s.text.clone(), s.ink.len(), false)], ops: std::iter::once(s.op).chain(s.extra_ops.iter().copied()).collect(), origin: Origin { tm: s.tm.0, @@ -1810,7 +1839,11 @@ fn reading_lines(doc: &Document, page: usize) -> Result<(Vec, bool), E let mut lines = w.lines; // A visual-order Indic font draws pre-base vowel signs before their consonant. for l in lines.iter_mut().filter(|l| l.visual_order && !l.actual_text) { + let shown = l.text.clone(); l.text = crate::indic::to_logical(&l.text); + if l.text != shown { + l.runs.clear(); + } } lines.retain(|l| !l.text.trim().is_empty()); Ok((lines, w.budget.truncated)) diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 735ba0c795..1ddfb7fe86 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5842,7 +5842,7 @@ status = "partial" commands = [] tools = ["doc_audit_redactions"] evidence = ["crates/audit/src/tests.rs::faux_redaction_is_found", "crates/audit/src/tests.rs::background_before_text_is_not_flagged", "crates/audit/src/tests.rs::light_shading_after_text_is_not_flagged", "crates/audit/src/tests.rs::graphics_state_carries_across_streams", "crates/audit/src/tests.rs::audit_fails_closed_on_undecodable_stream", "crates/audit/src/tests.rs::unapplied_redact_mark_is_found", "crates/audit/src/tests.rs::applied_redaction_at_the_end_of_a_line_is_not_flagged", "crates/audit/src/tests.rs::faux_redaction_at_the_end_of_a_line_is_found", "crates/audit/src/tests.rs::faux_redaction_over_a_short_value_after_a_long_label_is_found", "crates/audit/src/tests.rs::faux_redaction_over_mirrored_text_is_found", "crates/audit/src/tests.rs::faux_redaction_over_a_partial_value_after_a_long_label_is_found", "crates/audit/src/tests.rs::faux_redaction_over_a_partial_value_after_a_long_label_on_a_rotated_page_is_found", "crates/audit/src/tests.rs::faux_redactions_with_multiple_covers_lines_and_spacing_are_found", "crates/audit/src/tests.rs::a_cover_over_one_word_of_a_long_line_is_found", "crates/audit/src/tests.rs::a_box_grazing_a_word_is_not_flagged", "crates/automation/tests/it/automation.rs::auditing_faux_redactions_through_tools", "crates/automation/tests/it/automation.rs::an_applied_redaction_at_the_end_of_a_line_audits_clean", "crates/audit/src/tests.rs::rectangle_drawn_as_a_path_covers_text_like_re", "crates/audit/src/tests.rs::non_rectangular_path_is_not_a_cover", "apps/pdfcraft-cli/tests/it/audit.rs::audit_rectangles_match_the_tool_in_top_left_points", "crates/audit/src/tests.rs::rectangle_filled_after_restore_is_a_cover", "crates/audit/src/tests.rs::later_text_on_the_line_does_not_clear_an_earlier_cover", "crates/audit/src/tests.rs::indirect_subtype_is_resolved", "crates/audit/src/tests.rs::copy_restricted_documents_withhold_covered_text", "crates/audit/src/tests.rs::cover_and_line_comparisons_are_bounded", "crates/audit/src/tests.rs::covered_text_across_findings_is_bounded"] -notes = "New pdfcraft-audit crate. A filled rectangle (re + fill, or a single m/l subpath whose corners form an axis-aligned rectangle, #1510; opaque) counts as a cover only when painted after the start of the text line it overlaps and dark enough to hide it (WCAG contrast < 3.0); page backgrounds, table shading and highlights are not flagged. Coverage is measured on the words a line draws, not its box, so the gap an applied redaction leaves is not counted as hidden text; a cover over 40% of the line's box still counts when it has a positive-area intersection with one word, so a short value after a long label is found. A cover hiding half of any one word counts too (#2662), so a box over one number in a long line, or an unapplied mark over it, is found; a sliver under half of every word it touches and under 40% of the line is not flagged. Graphics state carries across a page's content streams. Fails closed: undecodable content, or more than 50 million cover-against-word comparisons on a page or 64 MiB of covered text in the report, is an error, never a clean bill. A document that withholds copying gets every finding with the covered text left out. Also flags /Redact annotations never applied. CLI: pdfcraft-cli audit (fails when findings exist). Tool and CLI report rectangles in points from the top-left of the displayed page, after /Rotate and rounded to 0.01 pt, like the other tools." +notes = "New pdfcraft-audit crate. A filled rectangle (re + fill, or a single m/l subpath whose corners form an axis-aligned rectangle, #1510; opaque) counts as a cover only when painted after the start of the text line it overlaps and dark enough to hide it (WCAG contrast < 3.0); page backgrounds, table shading and highlights are not flagged. Coverage is measured on the words a line draws, not its box, so the gap an applied redaction leaves is not counted as hidden text; a cover over 40% of the line's box still counts when it has a positive-area intersection with one word, so a short value after a long label is found. A cover hiding half of any one word counts too (#2662), so a box over one number in a long line, or an unapplied mark over it, is found; a sliver under half of every word it touches and under 40% of the line is not flagged. Graphics state carries across a page's content streams. Fails closed: undecodable content, or more than 50 million cover-against-word comparisons on a page or 64 MiB of covered text in the report, is an error, never a clean bill. A finding's covered text is the runs of the line whose words the cover overlaps, not the words of the line painted beside it (#4164); a line of one run, a right-to-left or visual-order line, or a cover over none of its words reports the whole line. A document that withholds copying gets every finding with the covered text left out. Also flags /Redact annotations never applied. CLI: pdfcraft-cli audit (fails when findings exist). Tool and CLI report rectangles in points from the top-left of the displayed page, after /Rotate and rounded to 0.01 pt, like the other tools." [[feature]] id = "protect.remove-hidden-on-close"