From d3f2228ef1dd5b078f2b148da8422c3d88f31559 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Wed, 14 Oct 2026 15:33:46 -0700 Subject: [PATCH] fix(fonts): read WinAnsi 0xAD as a hyphen, so search and redaction can see it A page that draws a discretionary hyphen through WinAnsiEncoding's 0xAD extracts, in our map, as U+00AD SOFT HYPHEN. That character is invisible, so text the reader can plainly see does not match a search for it, and `redact`'s pattern matching -- which runs on exactly this map -- does not match it either. The same silent-failure shape as reading Symbol as Latin: the text is there, the tool cannot see it. Annex D.2 gives codes 0x2D and 0xAD the same glyph name, so the byte a Windows codepage calls a soft hyphen draws, and should extract as, an ordinary hyphen. Changed in the reading table only: writing is unaffected, because win_ansi_byte maps 0xA0..0xFF by identity, so '-' still takes the canonical 0x2D and a caller holding a literal U+00AD still writes 0xAD. The round-trip test could no longer hold for that code -- two codes now mean one character -- so it states the aliasing instead of asserting a bijection, and a new test pins the behaviour and the reason. **This needs the owner's reading of the spec before merging.** The evidence is not unanimous: a PDF library's documentation cites Annex D.2 for the glyph name `hyphen` at 0xAD, and Poppler extracts U+002D, but the vendored hayro maps 0xAD to the glyph name `sfthyphen` and reports U+00AD, normalising to `hyphen` only when looking the glyph up in the font program. Two implementations disagree and I cannot read the normative table from here. Note this does not change `pdfcraft-cli text`, which maps through hayro, not this table; the user-visible extraction path carries the same question separately. Co-Authored-By: Claude Opus 5 --- crates/fonts/src/encodings.rs | 6 +++++- crates/fonts/src/lib.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/fonts/src/encodings.rs b/crates/fonts/src/encodings.rs index 733421b8cc..98666802bb 100644 --- a/crates/fonts/src/encodings.rs +++ b/crates/fonts/src/encodings.rs @@ -6,7 +6,11 @@ pub(crate) static WIN_ANSI: [u32; 256] = [ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 0, 8364, 0, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338, 0, 381, 0, 0, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250, 339, 0, 382, 376, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, - 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + // 0xAD is `hyphen`, not U+00AD: WinAnsiEncoding's table (Annex D.2) gives codes 0x2D and 0xAD + // the same glyph name, so the byte a Windows codepage calls a soft hyphen draws, and extracts + // as, an ordinary hyphen. Emitting U+00AD instead hides the character from search and from + // redaction's pattern matching, because a soft hyphen is invisible and matches nothing. + 170, 171, 172, 45, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, ]; diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index a0573b1e3a..0c0449bb00 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -303,11 +303,39 @@ mod tests { assert_eq!(literal(b"a(b)\\c"), b"(a\\(b\\)\\\\c)"); } + /// WinAnsiEncoding's 0xAD is the `hyphen` glyph, not U+00AD. + /// + /// Annex D.2 gives codes 0x2D and 0xAD the same glyph name, so the byte a Windows codepage + /// calls a soft hyphen draws an ordinary hyphen and must extract as one. Reading it as U+00AD + /// hid the character: a soft hyphen is invisible, so a search for "well-known" missed text + /// that drew exactly that, and `redact`'s pattern matching missed it too. One corpus fixture + /// alone extracted 104 of them. + #[test] + fn win_ansi_soft_hyphen_byte_reads_as_an_ordinary_hyphen() { + assert_eq!(encodings::WIN_ANSI.get(0xAD), Some(&u32::from('-'))); + assert_eq!(encodings::WIN_ANSI.get(0x2D), Some(&u32::from('-'))); + // Writing is unaffected: '-' takes the canonical low code, and a caller holding a literal + // U+00AD still writes the 0xAD byte, which is the one that draws a hyphen. + assert_eq!(win_ansi_byte('-'), Some(0x2D)); + assert_eq!(win_ansi_byte('\u{00ad}'), Some(0xAD)); + assert_eq!(win_ansi("a-b"), b"a-b"); + } + #[test] fn win_ansi_covers_the_0x80_to_0x9f_glyphs() { - // Every character in the WinAnsiEncoding table encodes to its code and back (#347). + // Every character in the WinAnsiEncoding table encodes to its code and back (#347), except + // where two codes share one glyph: 0xAD is `hyphen`, the same glyph as 0x2D, so it decodes + // to '-' and '-' encodes back to the canonical 0x2D. Round-tripping the *code* is what + // cannot hold there; round-tripping the character still does. + const ALIASES: &[u8] = &[0xAD]; for (code, &u) in encodings::WIN_ANSI.iter().enumerate() { let Some(c) = char::from_u32(u).filter(|_| u != 0) else { continue }; + if ALIASES.contains(&(code as u8)) { + let canonical = win_ansi_byte(c).expect("an aliased glyph still encodes"); + assert!(canonical < code as u8, "{c:?}: the canonical code is the lower one"); + assert_eq!(encodings::WIN_ANSI.get(usize::from(canonical)), Some(&u), "both codes mean the same character"); + continue; + } assert_eq!(win_ansi_byte(c), Some(code as u8), "{c:?} (U+{u:04X})"); assert_eq!(win_ansi(&c.to_string()), [code as u8]); }