diff --git a/ROADMAP.md b/ROADMAP.md index cb15663e8..c2a173416 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,6 +2,7 @@ **Stage: alpha** · next: beta, ~25% (ready for real work 50% → 75%) and ~400–700 h away +> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M2/gap 12: the Annex D.2 alias codes read as the ink they draw in both tables; the text oracle scores reading order; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-14 · **Last updated:** 2026-10-14 · **Change:** minor (M1: raw-deflate fallback preserves decoding-limit errors; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-13 · **Last updated:** 2026-10-14 · **Change:** minor (M14: the Italian catalog has every entry the other catalogs had and the code looks up; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-14 · **Last updated:** 2026-10-14 · **Change:** minor (M3 #2080 candidate: explicit search jumps join per-tab history; rail and control-channel Back/Forward; estimates unchanged) @@ -157,6 +158,9 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-15 (M1, #2267 refresh):** Integrated upstream `99941daa` while retaining the raw-deflate fallback correction and all upstream progress. Only independent documentation conflicts required resolution; focused production and test files are unchanged from the previously green CI head. Estimates unchanged. +- **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. +- **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. - **2026-10-12 (M6):** XFDF field import and data-file extraction join all text nodes in a value, preserving text after XML comments and processing instructions. Synthetic regressions cover Unicode, CDATA, empty values and PDF save/reopen. Estimates unchanged. - **2026-10-12 (M7, picture right-click menu):** The Edit text & images menu of a picture gains Cut Image, Redact Image (a redaction mark over the picture, applied from Redact a PDF), Create Link… and Recognize Text… (for that page), in all 21 catalogs. The menu was drawn only while the pointer was over the picture, so its items outside the picture (most of a logo's menu, and Delete below a page) could not be clicked; an open menu now stays drawn. Still missing from Acrobat's menu: Arrange (bring to front, send backward), Remove crop, Draw freehand and Edit image using… (opening an external editor). diff --git a/crates/fonts/src/encodings.rs b/crates/fonts/src/encodings.rs index 98666802b..88f0de5d1 100644 --- a/crates/fonts/src/encodings.rs +++ b/crates/fonts/src/encodings.rs @@ -5,11 +5,13 @@ pub(crate) static WIN_ANSI: [u32; 256] = [ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 0, 8364, 0, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338, - 0, 381, 0, 0, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250, 339, 0, 382, 376, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, - // 0xAD is `hyphen`, not U+00AD: WinAnsiEncoding's table (Annex D.2) gives codes 0x2D and 0xAD - // the same glyph name, so the byte a Windows codepage calls a soft hyphen draws, and extracts - // as, an ordinary hyphen. Emitting U+00AD instead hides the character from search and from - // redaction's pattern matching, because a soft hyphen is invisible and matches nothing. + // 0xA0 is `space` and 0xAD is `hyphen`, not U+00A0 and U+00AD: WinAnsiEncoding's table + // (Annex D.2) gives those codes the same glyph names as 0x20 and 0x2D, so the bytes a Windows + // codepage calls a non-breaking space and a soft hyphen draw, and extract as, an ordinary + // space and hyphen. Emitting the formatting characters instead hides them from search and + // from redaction's pattern matching: a soft hyphen is invisible and matches nothing, and a + // non-breaking space does not match the space a reader typed. + 0, 381, 0, 0, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250, 339, 0, 382, 376, 32, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 45, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, @@ -21,7 +23,8 @@ pub(crate) static MAC_ROMAN: [u32; 256] = [ 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 0, 196, 197, 199, 201, 209, 214, 220, 225, 224, 226, 228, 227, 229, 231, 233, 232, 234, 235, 237, 236, 238, 239, 241, 243, 242, 244, 246, 245, 250, 249, 251, 252, 8224, 176, 162, 163, 167, 8226, 182, 223, 174, 169, 8482, 180, 168, 8800, 198, 216, 8734, 177, 8804, 8805, 165, 181, 8706, 8721, 8719, 960, 8747, 170, 186, 937, 230, 248, 191, 161, 172, 8730, 402, - 8776, 8710, 171, 187, 8230, 160, 192, 195, 213, 338, 339, 8211, 8212, 8220, 8221, 8216, 8217, 247, 9674, 255, 376, 8260, 8364, 8249, 8250, 64257, + // 0xCA is `space`, the same glyph name Annex D.2 gives 0x20, for the reason WinAnsi 0xA0 is. + 8776, 8710, 171, 187, 8230, 32, 192, 195, 213, 338, 339, 8211, 8212, 8220, 8221, 8216, 8217, 247, 9674, 255, 376, 8260, 8364, 8249, 8250, 64257, 64258, 8225, 183, 8218, 8222, 8240, 194, 202, 193, 203, 200, 205, 206, 207, 204, 211, 212, 63743, 210, 218, 219, 217, 305, 710, 732, 175, 728, 729, 730, 184, 733, 731, 711, ]; diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index 73b4fbb08..cd0d8358e 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -305,31 +305,45 @@ mod tests { assert_eq!(literal(b"a(b)\\c"), b"(a\\(b\\)\\\\c)"); } - /// WinAnsiEncoding's 0xAD is the `hyphen` glyph, not U+00AD. + /// WinAnsiEncoding's 0xAD is the `hyphen` glyph and its 0xA0 is `space`, not U+00AD and + /// U+00A0; MacRomanEncoding's 0xCA is `space` too. /// - /// Annex D.2 gives codes 0x2D and 0xAD the same glyph name, so the byte a Windows codepage - /// calls a soft hyphen draws an ordinary hyphen and must extract as one. Reading it as U+00AD - /// hid the character: a soft hyphen is invisible, so a search for "well-known" missed text - /// that drew exactly that, and `redact`'s pattern matching missed it too. One corpus fixture - /// alone extracted 104 of them. + /// Annex D.2 gives each of those codes the same glyph name as the ordinary one (0x2D, 0x20), + /// so the bytes a Windows codepage calls a soft hyphen and a non-breaking space draw ordinary + /// ink and must extract as it. Reading them as the formatting characters hid them: a soft + /// hyphen is invisible, so a search for "well-known" missed text that drew exactly that, and + /// `redact`'s pattern matching missed it too; a non-breaking space does not match the space a + /// reader types. Over the 989-file pdf.js corpus this takes the soft hyphens we extract from + /// 117 to 13 (104 of them in `XiaoBiaoSong.pdf` alone) and the non-breaking spaces from 30 to + /// 26; what remains comes from `/ToUnicode` maps, which say so explicitly and are not touched. + /// pdftotext reads both codes as the ordinary character. #[test] - fn win_ansi_soft_hyphen_byte_reads_as_an_ordinary_hyphen() { + fn the_alias_codes_read_as_the_glyph_they_draw() { assert_eq!(encodings::WIN_ANSI.get(0xAD), Some(&u32::from('-'))); assert_eq!(encodings::WIN_ANSI.get(0x2D), Some(&u32::from('-'))); - // Writing is unaffected: '-' takes the canonical low code, and a caller holding a literal - // U+00AD still writes the 0xAD byte, which is the one that draws a hyphen. + assert_eq!(encodings::WIN_ANSI.get(0xA0), Some(&u32::from(' '))); + assert_eq!(encodings::WIN_ANSI.get(0x20), Some(&u32::from(' '))); + assert_eq!(encodings::MAC_ROMAN.get(0xCA), Some(&u32::from(' '))); + assert_eq!(encodings::MAC_ROMAN.get(0x20), Some(&u32::from(' '))); + // MacRoman has no soft hyphen; 0xAD there is `notequal`, and keeps it. + assert_eq!(encodings::MAC_ROMAN.get(0xAD), Some(&0x2260)); + // Writing is unaffected: '-' and ' ' take the canonical low codes, and a caller holding a + // literal U+00AD or U+00A0 still writes the high byte, which is the one that draws them. assert_eq!(win_ansi_byte('-'), Some(0x2D)); + assert_eq!(win_ansi_byte(' '), Some(0x20)); assert_eq!(win_ansi_byte('\u{00ad}'), Some(0xAD)); - assert_eq!(win_ansi("a-b"), b"a-b"); + assert_eq!(win_ansi_byte('\u{00a0}'), Some(0xA0)); + assert_eq!(win_ansi("a-b c"), b"a-b c"); } #[test] fn win_ansi_covers_the_0x80_to_0x9f_glyphs() { // Every character in the WinAnsiEncoding table encodes to its code and back (#347), except - // where two codes share one glyph: 0xAD is `hyphen`, the same glyph as 0x2D, so it decodes - // to '-' and '-' encodes back to the canonical 0x2D. Round-tripping the *code* is what + // where two codes share one glyph: 0xAD is `hyphen`, the same glyph as 0x2D, and 0xA0 is + // `space`, the same glyph as 0x20, so each decodes to the ordinary character and that + // character encodes back to the canonical low code. Round-tripping the *code* is what // cannot hold there; round-tripping the character still does. - const ALIASES: &[u8] = &[0xAD]; + const ALIASES: &[u8] = &[0xA0, 0xAD]; for (code, &u) in encodings::WIN_ANSI.iter().enumerate() { let Some(c) = char::from_u32(u).filter(|_| u != 0) else { continue }; if ALIASES.contains(&(code as u8)) { diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 3ad34eead..0cc456cba 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -1863,6 +1863,32 @@ mod encoding_tests { } } + /// The Annex D.2 alias codes read, through the whole decode path, as the ink they draw. + /// + /// This is the map `redact` matches its patterns on (`unicode_map`), so a code that decodes + /// to an invisible formatting character is a silent miss, not a cosmetic one: "well-known" + /// written with the 0xAD byte matched no search and no redaction pattern. + #[test] + fn the_alias_codes_decode_as_ordinary_ink_and_encode_back_to_the_low_code() { + let (doc, d) = simple("Helvetica", None, Some(Object::name("WinAnsiEncoding"))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.decode(b"well\xadknown"), "well-known"); + assert_eq!(m.decode(b"a\xa0b"), "a b"); + assert_eq!(m.encode("well-known a b").as_deref(), Some(&b"well-known a b"[..]), "the low code is the one written"); + + let (doc, d) = simple("Helvetica", None, Some(Object::name("MacRomanEncoding"))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.decode(b"a\xcab"), "a b"); + assert_eq!(m.decode(b"a\xadb"), "a\u{2260}b", "MacRoman 0xAD is `notequal`, not a hyphen"); + assert_eq!(m.encode("a b").as_deref(), Some(&b"a b"[..])); + + // An explicit /Differences name still says what it says: this is a property of those + // codes in those tables, not of the glyph names. + let (doc, d) = simple("Helvetica", None, Some(differences(vec![Object::Int(0x41), Object::name("sfthyphen"), Object::name("nbspace")]))); + let m = Metrics::from_dict(&doc, &d); + assert_eq!(m.decode(b"AB"), "\u{ad}\u{a0}"); + } + /// Pins a deliberate restraint, so it is not "fixed" into a regression: a great many ordinary /// Latin TrueType subsets set the `/FontDescriptor` symbolic flag and still rely on a Latin /// encoding. Suppressing the implicit table on the flag would lose real text on a large share diff --git a/crates/oracle/README.md b/crates/oracle/README.md index 7d2996b77..04eea193a 100644 --- a/crates/oracle/README.md +++ b/crates/oracle/README.md @@ -38,7 +38,7 @@ match pdfcraft_oracle::pdftotext() { |---|---|---| | `qpdf()` | qpdf | `--check` structural validation; the independent owner-password oracle | | `pdftoppm()` | Poppler | page pixels, for comparing against our own raster output | -| `pdftotext()` | Poppler | text extraction (`cargo xtask text-oracle` scores word F1 against it) | +| `pdftotext()` | Poppler | text extraction (`cargo xtask text-oracle` scores word F1 and reading order against it) | Each returns `Result`. `Oracle::command()` hands back a fresh `Command`; `Absent` renders as an actionable sentence naming the install command. @@ -92,6 +92,30 @@ picture through every corner of the format, so a correct decoder renders them al render 1 distinct image across all 96; Poppler renders 8, drawing a blank page for seven of them. `crates/render/tests/jbig2_invariants.rs` pins that invariant, and needs no oracle to do it. +## Comparing text + +`cargo xtask text-oracle` scores two different things against `pdftotext`, because one number +hides the other. + +**Word F1** is a bag of words: how much of what each side extracted the other also extracted, +with multiplicity. It catches a character read wrong, a glyph not read at all, a font we cannot +map. It is blind to *order* — a page emitted back to front scores 1.000. + +**Reading order** is the longest common subsequence of the two word sequences, as a share of the +longer one. It catches the failures F1 cannot see: a column read into the middle of its +neighbour, a figure's labels threaded through a paragraph, a form's value parted from its label. +Both sides are cut to the first 3,000 words, because the comparison is quadratic and three pages +is plenty of signal; the run says how many files that bit. + +The second score exists because of a measurement. Reviewing #2431, a change that moves text and +alters nothing else, word F1 called 26 of the 29 files whose output changed *identical*, while +the order score separated 18 improvements from 6 regressions — the regressions on forms, where +a checkbox's number was parted from its label. + +Read it the same way as the pixel score, and then once more: Poppler's reading order is a second +opinion, not the truth, and the same split can score as a win on one file and a loss on another +purely because `pdftotext` happens to make it in one place and not the other. + ## Still to do - Acrobat itself as the reference, rather than other open renderers. diff --git a/crates/render/src/text.rs b/crates/render/src/text.rs index ae5ffa9a9..b767e2831 100644 --- a/crates/render/src/text.rs +++ b/crates/render/src/text.rs @@ -1740,6 +1740,60 @@ mod tests { } } + /// One page per `(encoding, content)`, in Helvetica (`/F1`). + fn encoded_pages(pages: &[(lopdf::Object, &[u8])]) -> Pdf { + use lopdf::{Document, Object, Stream, dictionary}; + let mut doc = Document::with_version("1.7"); + let pages_id = doc.new_object_id(); + let mut kids = Vec::new(); + for (encoding, content) in pages { + let font = + doc.add_object(dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica", "Encoding" => encoding.clone() }); + let content = doc.add_object(Stream::new(dictionary! {}, content.to_vec())); + let page = doc.add_object(dictionary! { "Type" => "Page", "Parent" => pages_id, "MediaBox" => vec![Object::Integer(0), Object::Integer(0), Object::Integer(300), Object::Integer(300)], "Resources" => dictionary! {"Font" => dictionary! {"F1" => font}}, "Contents" => content }); + kids.push(Object::Reference(page)); + } + let count = kids.len() as i64; + doc.objects.insert(pages_id, dictionary! {"Type" => "Pages", "Kids" => kids, "Count" => count}.into()); + let catalog = doc.add_object(dictionary! {"Type" => "Catalog", "Pages" => pages_id}); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).unwrap(); + Pdf::new(std::sync::Arc::new(bytes)).unwrap() + } + + /// The Annex D.2 alias codes extract as the ink they draw, not as formatting characters. + /// + /// WinAnsiEncoding gives 0xAD and 0xA0 the glyph names `hyphen` and `space`, the names it + /// also gives 0x2D and 0x20, and MacRomanEncoding gives 0xCA `space`. Reporting the Adobe + /// Glyph List values U+00AD and U+00A0 for them hid visible text: a soft hyphen is invisible, + /// so "well-known" written that way matched no search and no redaction pattern. + /// `XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 of them. + /// + /// A `/Differences` entry naming `sfthyphen` or `nbspace` is a deliberate statement about one + /// code and keeps its glyph list value. pdftotext draws the same line: on this fixture it + /// reads the base-encoded codes as "a-b a b" and the named ones as U+00AD. + #[test] + fn the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space() { + use lopdf::{Object, dictionary}; + let name = |n: &str| Object::Name(n.as_bytes().to_vec()); + let differences = Object::Dictionary(dictionary! { + "Type" => "Encoding", + "BaseEncoding" => "WinAnsiEncoding", + "Differences" => vec![Object::Integer(88), name("sfthyphen"), Object::Integer(89), name("nbspace")], + }); + let pdf = encoded_pages(&[ + (name("WinAnsiEncoding"), &b"BT /F1 20 Tf 1 0 0 1 20 150 Tm (a\xadb\xa0c) Tj ET"[..]), + (name("MacRomanEncoding"), &b"BT /F1 20 Tf 1 0 0 1 20 150 Tm (a\xcab) Tj ET"[..]), + (differences, &b"BT /F1 20 Tf 1 0 0 1 20 150 Tm (aXbYc) Tj ET"[..]), + ]); + let texts = + |page| extract_page(&pdf, page, &InterpreterSettings::default()).unwrap().glyphs.iter().map(|g| g.text.clone()).collect::>(); + assert_eq!(texts(0), ["a", "-", "b", " ", "c"], "WinAnsi 0xAD and 0xA0"); + assert_eq!(texts(1), ["a", " ", "b"], "MacRoman 0xCA"); + assert_eq!(texts(2), ["a", "\u{ad}", "b", "\u{a0}", "c"], "a named glyph keeps its glyph list value"); + } + #[test] fn narrow_ligatures_keep_every_letter() { // Split evenly, this "ffi" put its two "f"s so close that the second was dropped as a diff --git a/docs/architecture.md b/docs/architecture.md index a226cc8de..aa91c36c4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,7 +101,7 @@ Every user-facing feature is reachable headlessly (`AGENTS.md` §3): `cargo xtask ci` runs fmt, clippy (`-D warnings`), tests, the layering check, the WASM check, `xtask assets` (asset licences and hashes), `xtask parity` (checklist claims must cite real commands, tools and tests) and `xtask deny` (dependency licences). Other tasks: `xtask fuzz`, `xtask corpus` and `xtask check` (robustness -sweep over the pinned pdf.js corpus), `xtask text-oracle` (word F1 against `pdftotext`), +sweep over the pinned pdf.js corpus), `xtask text-oracle` (word F1 and reading order against `pdftotext`), `xtask demo-pdf`, `xtask screenshots`, `xtask models` (OCR models), `xtask version`. Releases are built by `.github/workflows/release.yml` ([releasing.md](releasing.md)). diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 9a5f4d20d..d7230c096 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -86,7 +86,7 @@ features total). | PowerPoint (`.pptx`) | yes | slides per page: title from largest text, body text, images, tables (no slide overflow reflow / normAutofit; first page sets presentation dimensions) | basic | `create.export-pptx` partial | | RTF | yes | yes | basic | `create.export-rtf` | | HTML (single page, multiple pages) | yes | single file with headings, paragraphs, images, tables | basic | `create.export-html` partial | -| Plain text | yes | yes (reading order) | good (word-F1 0.98 vs `pdftotext`) | `create.export-text`, `xtask text-oracle` | +| Plain text | yes | yes (reading order) | good (word-F1 0.98 vs `pdftotext`; reading order is scored separately, because F1 cannot see it) | `create.export-text`, `xtask text-oracle` | | Accessible text (from tags) | yes | no | none | `create.export-accessible-text` planned | | PNG, JPEG, TIFF | yes | yes | good | `create.export-png`, `create.export-jpeg-tiff` | | JPEG 2000 | yes | no | none | `create.export-jpeg2000` planned | diff --git a/docs/gaps.md b/docs/gaps.md index 003d93deb..2682d090a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,5 +1,6 @@ # Where PdfCraft falls short of Acrobat Pro +> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (gap 12: reading order is scored, not just word F1) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-14 · **Last updated:** 2026-10-14 · **Change:** minor (M1: raw-deflate fallback preserves decoding-limit errors; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-12 · **Last updated:** 2026-10-13 · **Change:** minor (gap 14: Vietnamese catalog shipped; Hindi/Indonesian/Korean and Arabic RTL remain) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-13 · **Last updated:** 2026-10-13 · **Change:** minor (gap 6: #1910 Organize Pages opens on the current page; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -251,6 +252,17 @@ update the numbers. every appearance we *do* draw: over the same 250 files, shown gives mean 2.16% / p90 5.35% / 25 files within 0.1%, hidden gives 2.18% / 5.55% / 22, with `annotation-tx3.pdf` jumping to 45.64%. The biases nearly cancel; `--hide-annotations` tells the two apart for one file at a time. +- **The text score was blind to reading order (2026-10-15).** Word F1 is a bag of words, so a page + emitted back to front scores a perfect 1.000 — and reading order is most of what text extraction + has left to get wrong (#2387, #2586). `cargo xtask text-oracle` now reports a second score: the + longest common subsequence of the two word sequences over the longer one, both cut to 3,000 + words. The hole was not hypothetical. Measuring #2431, a change that moves text and alters + nothing else, **word F1 called 26 of the 29 files whose output changed identical**, where the + order score separated 18 improvements from 6 regressions and named their shape: forms, where a + checkbox's number was parted from its label (`prefilled_f1040.pdf` 0.672 → 0.481). Read it as a + disagreement, like the pixel score: the same split scored as a win on one file and a loss on + another purely because `pdftotext` makes it in one place and not the other. The corpus-wide + first reading has not been taken yet. - **Evidence:** text extraction (word-F1 0.98 against `pdftotext`) and now rendering (above); qpdf checks written output on every CI run since `PDFCRAFT_ORACLES_REQUIRED`. Acrobat captures must stay local in `plan/acrobat/` (clean-room, `AGENTS.md` §1.1). @@ -370,6 +382,7 @@ update the numbers. | Date | Change | Summary | |---|---|---| +| 2026-10-15 | minor | Gap 12: the text oracle scores reading order as well as word F1, which is a bag of words and cannot see order; the first measurement separated 18 improvements from 6 regressions in a change F1 called unchanged on 26 of 29 files. Estimates unchanged. | | 2026-10-14 | minor | M1: raw-deflate fallback preserves decoding-limit errors in strict and tolerant decoding; synthetic filter and COS stream regressions. Estimates unchanged. | | 2026-10-15 | patch | Gap 24: record localization and installers as partial (unusable Chinese, Telugu and Arabic; unsigned Linux packages) | | 2026-10-13 | minor | Gap 6: Organize Pages selects and reveals the current page on open, including after an earlier grid selection elsewhere (#1910). Estimates unchanged. | diff --git a/docs/roadmap.md b/docs/roadmap.md index d9470232b..fce71ecd0 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # PdfCraft roadmap: milestones and what's next -> **Last reviewed:** 2026-10-14 · **Last updated:** 2026-10-14 · **Change:** minor (M1: raw-deflate fallback preserves decoding-limit errors; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M1: raw-deflate fallback preserves decoding-limit errors; refreshed upstream; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-13 · **Last updated:** 2026-10-14 · **Change:** minor (M14: the Italian catalog has every entry the other catalogs had and the code looks up, 2,528 entries; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-12 · **Last updated:** 2026-10-13 · **Change:** minor (M2/M6: spot-colour images convert once per pixel value; widget placement and the Fields panel scale with the field count; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-14 · **Last updated:** 2026-10-14 · **Change:** minor (M3 candidate: coordinate control clicks on slow frames and coalesced text selection; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -97,7 +97,7 @@ total matches the parity estimate. | M | Milestone | Plan size (h) | Done | Remaining (h, 2026-10-10) | Notes | |---|---|---|---|---|---| | M0 | Skeleton: workspace, xtask gates, CI | 15–30 | 98% | 1–2 | GitHub workflow, `deny.toml` (licence audit of every dependency), parity checklist (826 features, `xtask parity`) done. Missing: remaining crate stubs, testkit/oracle crates | -| M1 | COS: filters, crypt, parser, xref, writer | 120–200 | 80% | 20–40 | Done:
- filters and crypt: every standard-security revision R2–R6 (RC4, AES-128/256), SASLprep, permissions, creating encryption;
- cos: parse and repair (including invalid xref object-number ranges), decoded-stream limits including unfiltered data, decrypt on load, re-encrypt on save, incremental and full writing.
Corpus: open/edit/save passes on 958 files, and all 7 password-protected files open.
Full saves now pack objects into compressed object streams. An unchanged incremental save of a classic table writes a zero-length cross-reference subsection instead of a section with no subsections (#1339). A full rewrite keeps unknown trailer entries, including objects reachable only from them (#1338). Fuzzing runs nightly (`xtask fuzz`). Raw-deflate fallback preserves decoding-limit errors, including through tolerant COS stream decoding. Missing: ≥ 250 tests, own image codecs | +| M1 | COS: filters, crypt, parser, xref, writer | 120–200 | 80% | 20–40 | Done:
- filters and crypt: every standard-security revision R2–R6 (RC4, AES-128/256), SASLprep, permissions, creating encryption;
- cos: parse and repair (including invalid xref object-number ranges), decoded-stream limits including unfiltered data, decrypt on load, re-encrypt on save, incremental and full writing.
Corpus: open/edit/save passes on 958 files, and all 7 password-protected files open.
Full saves now pack objects into compressed object streams. An unchanged incremental save of a classic table writes a zero-length cross-reference subsection instead of a section with no subsections (#1339). A full rewrite keeps unknown trailer entries, including objects reachable only from them (#1338). Fuzzing runs nightly (`xtask fuzz`). Raw-deflate fallback preserves decoding-limit errors, including through tolerant COS stream decoding. Refreshed against upstream `99941daa` (2026-10-15). Missing: ≥ 250 tests, own image codecs | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. Page marks, added content and print output retain small page-unit transform coefficients when serialized. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. The renderer keeps its one-point crop-size floor in physical points after UserUnit scaling, with regression coverage for crop origins, rotations and f32 reciprocal rounding. Existing unit policy and nearly-zero fallback are retained. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings | diff --git a/vendor/README.md b/vendor/README.md index fb6588c14..87f9d8682 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -9,7 +9,7 @@ A PdfCraft patch in `hayro-syntax::Page::base_dimensions` expresses the existing | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| -| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts` |; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable` | +| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space` | | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts` |; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable` | | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; (27) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts` |; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (27) `raster::tests::text_rendering_modes_other_than_invisible_draw_type3_glyphs_as_filled`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable` | | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs` |; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable` | diff --git a/vendor/hayro-interpret/src/font/mod.rs b/vendor/hayro-interpret/src/font/mod.rs index 5ccf5c48a..7c8c8d086 100644 --- a/vendor/hayro-interpret/src/font/mod.rs +++ b/vendor/hayro-interpret/src/font/mod.rs @@ -444,6 +444,29 @@ impl Encoding { Self::BuiltIn => None, } } + + /// PdfCraft patch: the glyph name an Annex D.2 *alias code* reads as when the glyph is taken + /// as text rather than looked up in a font program; `None` for every other code. + /// + /// Annex D.2 gives WinAnsiEncoding codes 0xA0 and 0xAD the glyph names `space` and `hyphen`, + /// the same names it gives 0x20 and 0x2D, and MacRomanEncoding 0xCA `space`. The tables above + /// call them `nbspace` and `sfthyphen`, whose Adobe Glyph List values U+00A0 and U+00AD are + /// *formatting* characters, so extraction hid ink the reader can see: a soft hyphen is + /// invisible, and text that drew one matched no search and no redaction pattern -- one corpus + /// file alone produced 104 of them -- while a non-breaking space did not match the space a + /// reader typed. pdftotext reads both codes as the ordinary character. + /// + /// Only the base table is aliased, in the text direction only. A `/Differences` entry naming + /// `sfthyphen` or `nbspace` is a deliberate statement and keeps its glyph list value, which + /// is what pdftotext does too; and glyph *lookup* is untouched, so a font program's own + /// `sfthyphen` program is still found first. + fn text_alias(&self, code: u8) -> Option<&'static str> { + match (self, code) { + (Self::WinAnsi, 0xA0) | (Self::MacRoman, 0xCA) => Some("space"), + (Self::WinAnsi, 0xAD) => Some("hyphen"), + _ => None, + } + } } /// The font stretch. diff --git a/vendor/hayro-interpret/src/font/standard_font.rs b/vendor/hayro-interpret/src/font/standard_font.rs index c73c75dd2..139454a0b 100644 --- a/vendor/hayro-interpret/src/font/standard_font.rs +++ b/vendor/hayro-interpret/src/font/standard_font.rs @@ -546,6 +546,14 @@ impl StandardKind { } pub(crate) fn char_code_to_unicode(&self, code: u8) -> Option { + // PdfCraft patch: an Annex D.2 alias code reads as the ordinary character; see + // `Encoding::text_alias`. `code_to_ps_name` is shared with glyph lookup and stays as it + // is, so the alias is applied here, after `/Differences` has had its say. + if !self.encodings.contains_key(&code) + && let Some(name) = self.encoding.text_alias(code) + { + return glyph_name_to_unicode(name); + } self.code_to_ps_name(code).and_then(glyph_name_to_unicode) } diff --git a/vendor/hayro-interpret/src/font/true_type.rs b/vendor/hayro-interpret/src/font/true_type.rs index 40f335903..b36266985 100644 --- a/vendor/hayro-interpret/src/font/true_type.rs +++ b/vendor/hayro-interpret/src/font/true_type.rs @@ -286,7 +286,13 @@ impl EmbeddedKind { self.differences .get(&code) .map(|s| s.as_str()) + // PdfCraft patch: an Annex D.2 alias code reads as the ordinary character; see + // `Encoding::text_alias`. + .or_else(|| self.encoding.text_alias(code)) .or_else(|| self.encoding.map_code(code)) + // The last resort reads the code as WinAnsi whatever the font said, so it takes the + // same aliasing. + .or_else(|| Encoding::WinAnsi.text_alias(code)) .or_else(|| win_ansi::get(code)) } diff --git a/vendor/hayro-interpret/src/font/type1.rs b/vendor/hayro-interpret/src/font/type1.rs index 32b5e3ddd..02f76bfb8 100644 --- a/vendor/hayro-interpret/src/font/type1.rs +++ b/vendor/hayro-interpret/src/font/type1.rs @@ -239,6 +239,14 @@ impl Type1Kind { } fn char_code_to_unicode(&self, code: u8) -> Option { + // PdfCraft patch: an Annex D.2 alias code reads as the ordinary character; see + // `Encoding::text_alias`. `code_to_ps_name` is shared with glyph lookup and stays as it + // is, so the alias is applied here, after `/Differences` has had its say. + if !self.encodings.contains_key(&code) + && let Some(name) = self.encoding.text_alias(code) + { + return glyph_name_to_unicode(name); + } self.code_to_ps_name(code).and_then(glyph_name_to_unicode) } } @@ -336,6 +344,14 @@ impl CffKind { } fn char_code_to_unicode(&self, code: u8) -> Option { + // PdfCraft patch: an Annex D.2 alias code reads as the ordinary character; see + // `Encoding::text_alias`. `code_to_ps_name` is shared with glyph lookup and stays as it + // is, so the alias is applied here, after `/Differences` has had its say. + if !self.encodings.contains_key(&code) + && let Some(name) = self.encoding.text_alias(code) + { + return glyph_name_to_unicode(name); + } self.code_to_ps_name(code).and_then(glyph_name_to_unicode) } } diff --git a/vendor/hayro-interpret/src/font/type3.rs b/vendor/hayro-interpret/src/font/type3.rs index ccfeeb766..324fa03df 100644 --- a/vendor/hayro-interpret/src/font/type3.rs +++ b/vendor/hayro-interpret/src/font/type3.rs @@ -108,6 +108,9 @@ impl<'a> Type3<'a> { .encodings .get(&code) .map(|s| s.as_str()) + // PdfCraft patch: an Annex D.2 alias code reads as the ordinary character; + // see `Encoding::text_alias`. + .or_else(|| self.encoding.text_alias(code)) .or_else(|| self.encoding.map_code(code)); name.and_then(crate::font::glyph_name_to_unicode) .or_else(|| (0x20..0x7f).contains(&code).then_some(code as char)) diff --git a/xtask/src/gates.rs b/xtask/src/gates.rs index 6591233e9..8e7db8cf0 100644 --- a/xtask/src/gates.rs +++ b/xtask/src/gates.rs @@ -196,8 +196,13 @@ pub fn check(args: &[String]) -> anyhow::Result<()> { } /// `cargo xtask text-oracle [--limit N]`: compare `pdfcraft-cli text` with poppler's `pdftotext` -/// (external oracle process, never linked) over the corpus. Reports word-level F1 per file and the -/// median. Plan target (M2): median ≥ 0.97. +/// (external oracle process, never linked) over the corpus. +/// +/// Two scores per file, because one hides the other. **Word F1** ([`f1`]) is a bag of words: it +/// catches a character read wrong, a glyph not read at all, a font we cannot map, and is blind to +/// order. **Reading order** ([`order_score`]) catches what F1 cannot see — a column read into the +/// middle of its neighbour, a figure's labels threaded through a paragraph. Reports both; gates +/// neither. Plan target (M2): median F1 ≥ 0.97. pub fn text_oracle(args: &[String]) -> anyhow::Result<()> { let limit: usize = args.iter().position(|a| a == "--limit").and_then(|i| args.get(i + 1)).and_then(|v| v.parse().ok()).unwrap_or(usize::MAX); let pdftotext = match pdfcraft_oracle::pdftotext() { @@ -211,8 +216,11 @@ pub fn text_oracle(args: &[String]) -> anyhow::Result<()> { files.sort(); let cli = release_cli(); let mut scores = Vec::new(); + let mut orders = Vec::new(); let mut worst: Vec<(f64, String)> = Vec::new(); + let mut worst_order: Vec<(f64, String)> = Vec::new(); let mut not_permitted = 0usize; + let mut truncated = 0usize; for f in files.iter().take(limit) { // A document may forbid copying its text, and we honour that: `pdfcraft-cli text` refuses // and says how to override it with the owner password. pdftotext ignores the flag and @@ -225,29 +233,49 @@ pub fn text_oracle(args: &[String]) -> anyhow::Result<()> { let ours = run_capture(Command::new(&cli).arg("text").arg(f), 20); let theirs = run_capture(pdftotext.command().arg("-q").arg(f).arg("-"), 20); let (Some(ours), Some(theirs)) = (ours, theirs) else { continue }; - let (a, b) = (words(&ours), words(&theirs)); + let (sa, sb) = (word_sequence(&ours), word_sequence(&theirs)); + let (a, b) = (words(&sa), words(&sb)); if b.len() < 20 { continue; // image-only or near-empty per the oracle } let score = f1(&a, &b); + let (order, cut) = order_score(&sa, &sb); + truncated = truncated.saturating_add(usize::from(cut)); + let name = f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); scores.push(score); - worst.push((score, f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default())); + orders.push(order); + worst.push((score, name.clone())); + worst_order.push((order, name)); } if scores.is_empty() { bail!("no comparable files"); } - scores.sort_by(f64::total_cmp); + let summary = |v: &mut Vec| { + v.sort_by(f64::total_cmp); + let mean = v.iter().sum::() / v.len() as f64; + (v[v.len() / 2], mean, v[v.len() / 10]) + }; + let compared = scores.len(); + let (median, mean, p10) = summary(&mut scores); + let (omedian, omean, op10) = summary(&mut orders); worst.sort_by(|a, b| a.0.total_cmp(&b.0)); - let median = scores[scores.len() / 2]; - let mean = scores.iter().sum::() / scores.len() as f64; - println!("text oracle: {} files with text; median F1 {:.3}, mean {:.3}, p10 {:.3}", scores.len(), median, mean, scores[scores.len() / 10]); + worst_order.sort_by(|a, b| a.0.total_cmp(&b.0)); + println!("text oracle: {compared} files with text; median F1 {median:.3}, mean {mean:.3}, p10 {p10:.3}"); + println!( + " reading order: median {omedian:.3}, mean {omean:.3}, p10 {op10:.3}{}", + if truncated > 0 { format!(" ({truncated} cut to {MAX_ORDER_WORDS} words)") } else { String::new() } + ); if not_permitted > 0 { println!(" {not_permitted} skipped: the document forbids copying its text, which we honour and pdftotext does not"); } - println!("lowest:"); + println!("lowest F1:"); for (s, n) in worst.iter().take(12) { println!(" {s:.3} {n}"); } + println!("lowest reading order:"); + for (s, n) in worst_order.iter().take(12) { + println!(" {s:.3} {n}"); + } Ok(()) } @@ -416,14 +444,57 @@ fn run_capture_bytes(cmd: &mut Command, timeout_s: u64) -> Option> { Some(out) } -fn words(s: &str) -> BTreeMap { +/// The words of `s`, in the order they were extracted. +fn word_sequence(s: &str) -> Vec { + s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_lowercase).collect() +} + +fn words(seq: &[String]) -> BTreeMap { let mut m = BTreeMap::new(); - for w in s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()) { - *m.entry(w.to_lowercase()).or_insert(0) += 1; + for w in seq { + *m.entry(w.clone()).or_insert(0) += 1; } m } +/// Both word sequences are cut to this length before [`order_score`] compares them: the +/// comparison is quadratic, and three pages of prose is plenty of signal about reading order. +const MAX_ORDER_WORDS: usize = 3000; + +/// How much of the longer word sequence reads in the same order in both, and whether either side +/// was cut to [`MAX_ORDER_WORDS`] first. +/// +/// [`f1`] is a bag of words and cannot see order at all: a page emitted back to front scores a +/// perfect 1.000 against the page read properly. Reading order is most of what extraction has +/// left to get +/// wrong — a column read into the middle of its neighbour, a figure's labels threaded through a +/// paragraph, a form's value parted from its label — so it needs a number of its own. Reviewing +/// #2431, a change that moves text and alters nothing else, F1 called 26 of the 29 files whose +/// output changed identical, while this separated 18 improvements from 6 regressions. +/// +/// The score is the length of the longest common subsequence over the longer side. Like every +/// other oracle number here it is a *disagreement* with Poppler, not a verdict: the same split +/// scores as a win on one file and a loss on another purely because `pdftotext` happens to make +/// it in one place and not the other. +fn order_score(a: &[String], b: &[String]) -> (f64, bool) { + let cut = a.len() > MAX_ORDER_WORDS || b.len() > MAX_ORDER_WORDS; + let a = a.get(..a.len().min(MAX_ORDER_WORDS)).unwrap_or(a); + let b = b.get(..b.len().min(MAX_ORDER_WORDS)).unwrap_or(b); + let (n, m) = (a.len(), b.len()); + if n == 0 || m == 0 { + return (0.0, cut); + } + // Two rows of the usual table: after row `i`, `prev[j]` answers `a[..i]` against `b[..j]`. + let (mut prev, mut cur) = (vec![0u32; m + 1], vec![0u32; m + 1]); + for x in a { + for (j, y) in b.iter().enumerate() { + cur[j + 1] = if x == y { prev[j].saturating_add(1) } else { cur[j].max(prev[j + 1]) }; + } + std::mem::swap(&mut prev, &mut cur); + } + (f64::from(prev[m]) / n.max(m) as f64, cut) +} + fn f1(a: &BTreeMap, b: &BTreeMap) -> f64 { let common: usize = a.iter().map(|(w, n)| (*n).min(*b.get(w).unwrap_or(&0))).sum(); let (na, nb): (usize, usize) = (a.values().sum(), b.values().sum()); @@ -436,7 +507,42 @@ fn f1(a: &BTreeMap, b: &BTreeMap) -> f64 { #[cfg(test)] mod tests { - use super::root; + use super::{f1, order_score, root, word_sequence, words}; + + /// The two scores answer different questions, and the point of adding the second one is the + /// first row: F1 cannot tell a page from the same page read backwards. + #[test] + fn reading_order_sees_what_word_f1_cannot() { + let score = |ours: &str, theirs: &str| { + let (a, b) = (word_sequence(ours), word_sequence(theirs)); + (f1(&words(&a), &words(&b)), order_score(&a, &b).0) + }; + + // Two columns, read as columns and read interleaved: every word is there either way. + let columns = "left one left two left three right one right two right three"; + let woven = "left one right one left two right two left three right three"; + let (same_words, out_of_order) = score(woven, columns); + assert_eq!(same_words, 1.0, "F1 is a bag of words and cannot see the difference"); + assert!(out_of_order < 0.7, "reading order can: {out_of_order}"); + + // It is not blind to the thing F1 measures, either: a missing word costs both. + let (f, o) = score("left one left two left", columns); + assert!(f < 0.9 && o < 0.9, "{f} {o}"); + + // Identical text, and the degenerate inputs. + assert_eq!(score(columns, columns), (1.0, 1.0)); + assert_eq!(score("", columns).1, 0.0); + assert_eq!(score(columns, "").1, 0.0); + assert_eq!(order_score(&[], &[]), (0.0, false)); + + // Case and punctuation are not order: both sides are folded the same way. + assert_eq!(score("Left, one. LEFT two!", "left one left two"), (1.0, 1.0)); + + // Over the cap, the comparison still answers, and says that it was cut. + let long: Vec = (0..super::MAX_ORDER_WORDS + 10).map(|i| i.to_string()).collect(); + let (s, cut) = order_score(&long, &long); + assert!((s - 1.0).abs() < 1e-9 && cut, "{s} {cut}"); + } // Issue #211: Finder lists the app under Open With > Recommended only when the // bundle claims PDFs by UTI (#176).