diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 42659f9f4b..a805648e28 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -51,16 +51,25 @@ impl PdfString { } /// Decode a text string: UTF-16BE (BOM), UTF-8 (BOM, PDF 2.0) or PDFDocEncoding. + /// + /// Lenient beyond the spec, as readers are: invalid sequences after a BOM decode lossily + /// instead of dropping the string, a BOM-less string whose bytes are valid UTF-8 with + /// non-ASCII bytes (written by some producers) is read as UTF-8 rather than as + /// PDFDocEncoding mojibake, and stray leading U+FEFF (doubled BOMs) are removed. pub fn to_text(&self) -> String { - let b = &self.bytes; - if b.len() >= 2 && b[0] == 0xFE && b[1] == 0xFF { - let units: Vec = b[2..].as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); - return String::from_utf16_lossy(&units); - } - if b.len() >= 3 && b[..3] == [0xEF, 0xBB, 0xBF] { - return String::from_utf8_lossy(&b[3..]).into_owned(); - } - b.iter().map(|&c| pdfdoc_char(c)).collect() + let s = match self.bytes.as_slice() { + [0xFE, 0xFF, rest @ ..] => { + let units: Vec = rest.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) + } + [0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest).into_owned(), + b => match std::str::from_utf8(b) { + // Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch. + Ok(utf8) if !b.is_ascii() => utf8.to_owned(), + _ => b.iter().map(|&c| pdfdoc_char(c)).collect(), + }, + }; + if s.starts_with('\u{FEFF}') { s.trim_start_matches('\u{FEFF}').to_owned() } else { s } } } @@ -384,6 +393,31 @@ mod tests { assert!(!PdfString::text("abc").bytes.starts_with(&[0xFE, 0xFF])); } + #[test] + fn text_string_encodings() { + // UTF-16BE with BOM. + let mut utf16 = vec![0xFE, 0xFF]; + utf16.extend("第一章 概述".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(PdfString::literal(utf16).to_text(), "第一章 概述"); + // UTF-8 with BOM (PDF 2.0): the BOM is not part of the text. + let mut utf8 = vec![0xEF, 0xBB, 0xBF]; + utf8.extend_from_slice("第一章".as_bytes()); + assert_eq!(PdfString::literal(utf8).to_text(), "第一章"); + // Invalid UTF-8 after the BOM decodes lossily instead of vanishing. + assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, b'A', 0xFF, b'B']).to_text(), "A\u{FFFD}B"); + // BOM-less raw UTF-8 from producers that skip the BOM. + assert_eq!(PdfString::literal("目录".as_bytes()).to_text(), "目录"); + // PDFDocEncoding Latin-1 (not valid UTF-8) stays PDFDocEncoding. + assert_eq!(PdfString::literal(vec![b'C', b'a', b'f', 0xE9]).to_text(), "Café"); + // Doubled BOMs leave no U+FEFF behind. + assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A']).to_text(), "A"); + assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF, b'A']).to_text(), "A"); + // Odd trailing byte and empty strings never panic. + assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0x00, b'A', 0x00]).to_text(), "A"); + assert_eq!(PdfString::literal(vec![0xFE, 0xFF]).to_text(), ""); + assert_eq!(PdfString::literal(Vec::new()).to_text(), ""); + } + #[test] fn pdfdoc_encoding_high_range() { assert_eq!(PdfString::literal(vec![0x84, 0x92, 0xA0, b'A']).to_text(), "—™€A"); diff --git a/crates/render/src/inspect.rs b/crates/render/src/inspect.rs index fa744a8ac4..ada3ae7e4f 100644 --- a/crates/render/src/inspect.rs +++ b/crates/render/src/inspect.rs @@ -520,7 +520,7 @@ impl<'a> Inspector<'a> { fn text(&self, d: &Dictionary, key: &[u8]) -> Option { let o = self.resolve(d.get(key).ok()?); let s = match o { - Object::String(..) => lopdf::decode_text_string(o).ok()?, + Object::String(bytes, _) => text_string(bytes), Object::Name(n) => String::from_utf8_lossy(n).into_owned(), _ => return None, }; @@ -605,7 +605,10 @@ impl<'a> Inspector<'a> { if let Ok(Object::Array(pairs)) = node.get(b"Names").map(|o| self.resolve(o)) { for pair in pairs.chunks(2) { if let [k, v] = pair { - let key = lopdf::decode_text_string(self.resolve(k)).unwrap_or_default(); + let key = match self.resolve(k) { + Object::String(bytes, _) => text_string(bytes), + _ => String::new(), + }; out.push((key, v)); } } @@ -965,8 +968,16 @@ impl<'a> Inspector<'a> { }; let value = match d.get(b"V").map(|v| self.resolve(v)) { Ok(Object::Name(n)) => Some(String::from_utf8_lossy(n).into_owned()), - Ok(v @ Object::String(..)) => lopdf::decode_text_string(v).ok(), - Ok(Object::Array(a)) => Some(a.iter().filter_map(|x| lopdf::decode_text_string(self.resolve(x)).ok()).collect::>().join(", ")), + Ok(Object::String(bytes, _)) => Some(text_string(bytes)), + Ok(Object::Array(a)) => Some( + a.iter() + .filter_map(|x| match self.resolve(x) { + Object::String(bytes, _) => Some(text_string(bytes)), + _ => None, + }) + .collect::>() + .join(", "), + ), Ok(Object::Dictionary(_)) if kind == FieldKind::Signature => Some("signed".into()), _ => None, }; @@ -1112,6 +1123,12 @@ impl<'a> Inspector<'a> { } } +/// A text string's value, decoded by the same rules as the engine's object model +/// ([`pdfcraft_cos::PdfString::to_text`]) so the panels and outline editing agree. +fn text_string(bytes: &[u8]) -> String { + pdfcraft_cos::PdfString::literal(bytes).to_text() +} + fn rect4(o: &Object) -> [f32; 4] { match o { Object::Array(a) if a.len() == 4 => { @@ -1651,6 +1668,29 @@ trailer << /Root 1 0 R >> ); } + #[test] + fn outline_titles_decode_every_text_string_encoding() { + // Issue #142: CJK bookmark titles showed as mojibake or with a stray BOM. + let pdf = b"%PDF-1.7 +1 0 obj << /Type /Catalog /Pages 2 0 R /Outlines 10 0 R >> endobj +2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 400] >> endobj +3 0 obj << /Type /Page /Parent 2 0 R >> endobj +10 0 obj << /Type /Outlines /First 11 0 R /Last 16 0 R /Count 6 >> endobj +11 0 obj << /Title /Parent 10 0 R /Next 12 0 R /Dest [3 0 R /Fit] >> endobj +12 0 obj << /Title /Parent 10 0 R /Next 13 0 R /Dest [3 0 R /Fit] >> endobj +13 0 obj << /Title /Parent 10 0 R /Next 14 0 R /Dest [3 0 R /Fit] >> endobj +14 0 obj << /Title /Parent 10 0 R /Next 15 0 R /Dest [3 0 R /Fit] >> endobj +15 0 obj << /Title <436166E9> /Parent 10 0 R /Next 16 0 R /Dest [3 0 R /Fit] >> endobj +16 0 obj << /Title /Parent 10 0 R /Dest [3 0 R /Fit] >> endobj +trailer << /Root 1 0 R >> +%%EOF"; + let info = inspect(Arc::new(pdf.to_vec()), None).expect("opens"); + let titles: Vec<_> = info.outline.iter().map(|o| o.title.as_str()).collect(); + // UTF-16BE, UTF-8 with BOM, invalid UTF-8 after a BOM (lossy, not dropped), BOM-less + // raw UTF-8, PDFDocEncoding Latin-1, and a doubled BOM. + assert_eq!(titles, ["第一章", "目录", "A\u{FFFD}B", "概述", "Café", "A"]); + } + #[test] fn destination_points_map_to_the_displayed_page() { // A crop box away from the origin: x 100..400, y 50..450.