aleleba/pdfcraft
1
0
Fork 0
mirror of https://github.com/storytold/pdfcraft.git synced 2026-10-30 13:08:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

Refs #142: decode outline titles with BOMs and raw UTF-8 consistently (#614)

Browse Source 
The bookmarks panel decoded /Title through lopdf::decode_text_string,
which keeps a PDF 2.0 UTF-8 BOM as a visible U+FEFF, drops the whole
title when the bytes after a UTF-8 BOM are not valid UTF-8, and reads
BOM-less raw UTF-8 (written by some producers) as PDFDocEncoding
mojibake. Outline editing in pdfcraft-organize used a second decoder,
PdfString::to_text, so the two could disagree.

PdfString::to_text now decodes UTF-8 after a BOM lossily, reads
BOM-less strings that are valid non-ASCII UTF-8 as UTF-8, and strips
stray leading U+FEFF. The inspector's text strings (titles, name-tree
keys, field values) go through the same function.

The reporter's file decodes correctly on main, so this is hardening of
the decoder rather than a reproduction of that file.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-09 16:52:07 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent d483d6d475
commit 90bb2f14ad
2 changed files with 87 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/cos/src/object.rs
+43 -9
View File
@@ -51,16 +51,25 @@ impl PdfString {
}
/// Decode a text string: UTF-16BE (BOM), UTF-8 (BOM, PDF 2.0) or PDFDocEncoding.
///
/// Lenient beyond the spec, as readers are: invalid sequences after a BOM decode lossily
/// instead of dropping the string, a BOM-less string whose bytes are valid UTF-8 with
/// non-ASCII bytes (written by some producers) is read as UTF-8 rather than as
/// PDFDocEncoding mojibake, and stray leading U+FEFF (doubled BOMs) are removed.
pub fn to_text(&self) -> String {
let b = &self.bytes;
if b.len() >= 2 && b[0] == 0xFE && b[1] == 0xFF {
let units: Vec<u16> = b[2..].as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect();
return String::from_utf16_lossy(&units);
}
if b.len() >= 3 && b[..3] == [0xEF, 0xBB, 0xBF] {
return String::from_utf8_lossy(&b[3..]).into_owned();
}
b.iter().map(|&c| pdfdoc_char(c)).collect()
let s = match self.bytes.as_slice() {
[0xFE, 0xFF, rest @ ..] => {
let units: Vec<u16> = rest.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect();
String::from_utf16_lossy(&units)
}
[0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest).into_owned(),
b => match std::str::from_utf8(b) {
// Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch.
Ok(utf8) if !b.is_ascii() => utf8.to_owned(),
_ => b.iter().map(|&c| pdfdoc_char(c)).collect(),
},
};
if s.starts_with('\u{FEFF}') { s.trim_start_matches('\u{FEFF}').to_owned() } else { s }
}
}
@@ -384,6 +393,31 @@ mod tests {
assert!(!PdfString::text("abc").bytes.starts_with(&[0xFE, 0xFF]));
}
#[test]
fn text_string_encodings() {
// UTF-16BE with BOM.
let mut utf16 = vec![0xFE, 0xFF];
utf16.extend("第一章 概述".encode_utf16().flat_map(u16::to_be_bytes));
assert_eq!(PdfString::literal(utf16).to_text(), "第一章 概述");
// UTF-8 with BOM (PDF 2.0): the BOM is not part of the text.
let mut utf8 = vec![0xEF, 0xBB, 0xBF];
utf8.extend_from_slice("第一章".as_bytes());
assert_eq!(PdfString::literal(utf8).to_text(), "第一章");
// Invalid UTF-8 after the BOM decodes lossily instead of vanishing.
assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, b'A', 0xFF, b'B']).to_text(), "A\u{FFFD}B");
// BOM-less raw UTF-8 from producers that skip the BOM.
assert_eq!(PdfString::literal("目录".as_bytes()).to_text(), "目录");
// PDFDocEncoding Latin-1 (not valid UTF-8) stays PDFDocEncoding.
assert_eq!(PdfString::literal(vec![b'C', b'a', b'f', 0xE9]).to_text(), "Café");
// Doubled BOMs leave no U+FEFF behind.
assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A']).to_text(), "A");
assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF, b'A']).to_text(), "A");
// Odd trailing byte and empty strings never panic.
assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0x00, b'A', 0x00]).to_text(), "A");
assert_eq!(PdfString::literal(vec![0xFE, 0xFF]).to_text(), "");
assert_eq!(PdfString::literal(Vec::new()).to_text(), "");
}
#[test]
fn pdfdoc_encoding_high_range() {
assert_eq!(PdfString::literal(vec![0x84, 0x92, 0xA0, b'A']).to_text(), "—™€A");
crates/render/src/inspect.rs
+44 -4
View File
@@ -520,7 +520,7 @@ impl<'a> Inspector<'a> {
fn text(&self, d: &Dictionary, key: &[u8]) -> Option<String> {
let o = self.resolve(d.get(key).ok()?);
let s = match o {
Object::String(..) => lopdf::decode_text_string(o).ok()?,
Object::String(bytes, _) => text_string(bytes),
Object::Name(n) => String::from_utf8_lossy(n).into_owned(),
_ => return None,
};
@@ -605,7 +605,10 @@ impl<'a> Inspector<'a> {
if let Ok(Object::Array(pairs)) = node.get(b"Names").map(|o| self.resolve(o)) {
for pair in pairs.chunks(2) {
if let [k, v] = pair {
let key = lopdf::decode_text_string(self.resolve(k)).unwrap_or_default();
let key = match self.resolve(k) {
Object::String(bytes, _) => text_string(bytes),
_ => String::new(),
};
out.push((key, v));
}
}
@@ -965,8 +968,16 @@ impl<'a> Inspector<'a> {
};
let value = match d.get(b"V").map(|v| self.resolve(v)) {
Ok(Object::Name(n)) => Some(String::from_utf8_lossy(n).into_owned()),
Ok(v @ Object::String(..)) => lopdf::decode_text_string(v).ok(),
Ok(Object::Array(a)) => Some(a.iter().filter_map(|x| lopdf::decode_text_string(self.resolve(x)).ok()).collect::<Vec<_>>().join(", ")),
Ok(Object::String(bytes, _)) => Some(text_string(bytes)),
Ok(Object::Array(a)) => Some(
a.iter()
.filter_map(|x| match self.resolve(x) {
Object::String(bytes, _) => Some(text_string(bytes)),
_ => None,
})
.collect::<Vec<_>>()
.join(", "),
),
Ok(Object::Dictionary(_)) if kind == FieldKind::Signature => Some("signed".into()),
_ => None,
};
@@ -1112,6 +1123,12 @@ impl<'a> Inspector<'a> {
}
}
/// A text string's value, decoded by the same rules as the engine's object model
/// ([`pdfcraft_cos::PdfString::to_text`]) so the panels and outline editing agree.
fn text_string(bytes: &[u8]) -> String {
pdfcraft_cos::PdfString::literal(bytes).to_text()
}
fn rect4(o: &Object) -> [f32; 4] {
match o {
Object::Array(a) if a.len() == 4 => {
@@ -1651,6 +1668,29 @@ trailer << /Root 1 0 R >>
);
}
#[test]
fn outline_titles_decode_every_text_string_encoding() {
// Issue #142: CJK bookmark titles showed as mojibake or with a stray BOM.
let pdf = b"%PDF-1.7
1 0 obj << /Type /Catalog /Pages 2 0 R /Outlines 10 0 R >> endobj
2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 400] >> endobj
3 0 obj << /Type /Page /Parent 2 0 R >> endobj
10 0 obj << /Type /Outlines /First 11 0 R /Last 16 0 R /Count 6 >> endobj
11 0 obj << /Title <FEFF7B2C4E007AE0> /Parent 10 0 R /Next 12 0 R /Dest [3 0 R /Fit] >> endobj
12 0 obj << /Title <EFBBBFE79BAEE5BD95> /Parent 10 0 R /Next 13 0 R /Dest [3 0 R /Fit] >> endobj
13 0 obj << /Title <EFBBBF41FF42> /Parent 10 0 R /Next 14 0 R /Dest [3 0 R /Fit] >> endobj
14 0 obj << /Title <E6A682E8BFB0> /Parent 10 0 R /Next 15 0 R /Dest [3 0 R /Fit] >> endobj
15 0 obj << /Title <436166E9> /Parent 10 0 R /Next 16 0 R /Dest [3 0 R /Fit] >> endobj
16 0 obj << /Title <FEFFFEFF0041> /Parent 10 0 R /Dest [3 0 R /Fit] >> endobj
trailer << /Root 1 0 R >>
%%EOF";
let info = inspect(Arc::new(pdf.to_vec()), None).expect("opens");
let titles: Vec<_> = info.outline.iter().map(|o| o.title.as_str()).collect();
// UTF-16BE, UTF-8 with BOM, invalid UTF-8 after a BOM (lossy, not dropped), BOM-less
// raw UTF-8, PDFDocEncoding Latin-1, and a doubled BOM.
assert_eq!(titles, ["第一章", "目录", "A\u{FFFD}B", "概述", "Café", "A"]);
}
#[test]
fn destination_points_map_to_the_displayed_page() {
// A crop box away from the origin: x 100..400, y 50..450.