From 0375bd1e28960f15fc8947409d896d1fa52e36c9 Mon Sep 17 00:00:00 2001 From: Chen Tao Date: Sat, 10 Oct 2026 01:59:33 +0900 Subject: [PATCH] M6.2: form fields in Unicode CID fonts show Japanese instead of "?" (#535) * M6.2: form fields in Unicode CID fonts show Japanese instead of "?" Japanese forms often give their fields a /DA font that /DR defines as a Type0 font with a predefined Unicode CMap, such as a non-embedded HeiseiMin-W3 with /UniJIS-UCS2-H. Field appearances accepted only simple fonts, so they fell back to Helvetica and WinAnsi, and every Japanese character was drawn as "?". Appearances now use such fonts and encode the text in the CMap's form (UCS-2, UTF-16, UTF-8 or UTF-32), measuring full-width characters as one em so centring, auto size, comb cells and multiline wrapping fit. Identity-H and embedded CMaps still fall back to Helvetica. Co-Authored-By: Claude Opus 5.5 * M6.2: vertical Unicode CMaps fall back to Helvetica, since field appearances are horizontal --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Brandon Thomas --- crates/fonts/src/lib.rs | 115 ++++++++++++++++++++++++++++++++- crates/forms/src/appearance.rs | 63 +++++++++++++----- crates/forms/src/tests.rs | 49 ++++++++++++++ 3 files changed, 209 insertions(+), 18 deletions(-) diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index d5965f8bde..ecd7630bfb 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -39,15 +39,51 @@ pub fn helvetica_width(s: &str, size: f64) -> f64 { units * size / 1000.0 } +/// Approximate advance of `s` at `size` points in a CJK font: full-width characters take one +/// em, half-width forms half of one, and everything else is measured as Helvetica. +pub fn cjk_width(s: &str, size: f64) -> f64 { + let mut buf = [0u8; 4]; + s.chars() + .map(|c| match c { + '\u{ff61}'..='\u{ffdc}' | '\u{ffe8}'..='\u{ffee}' => size * 0.5, + _ if is_wide(c) => size, + _ => helvetica_width(c.encode_utf8(&mut buf), size), + }) + .sum() +} + +/// East Asian wide and full-width characters (Unicode UAX #11 W and F, by block). +fn is_wide(c: char) -> bool { + matches!(c, + '\u{1100}'..='\u{115f}' // Hangul Jamo initials + | '\u{2e80}'..='\u{303e}' // CJK radicals, Kangxi, ideographic description, CJK symbols + | '\u{3041}'..='\u{33ff}' // kana, Bopomofo, Hangul compatibility, Kanbun, CJK compatibility + | '\u{3400}'..='\u{4dbf}' // CJK extension A + | '\u{4e00}'..='\u{9fff}' // CJK unified ideographs + | '\u{a000}'..='\u{a4cf}' // Yi + | '\u{ac00}'..='\u{d7a3}' // Hangul syllables + | '\u{f900}'..='\u{faff}' // CJK compatibility ideographs + | '\u{fe30}'..='\u{fe4f}' // CJK compatibility forms + | '\u{ff00}'..='\u{ff60}' // full-width forms + | '\u{ffe0}'..='\u{ffe6}' + | '\u{20000}'..='\u{3fffd}' // supplementary ideographic planes + ) +} + /// Greedy line breaking within `width` points (paragraphs split on newlines; words longer /// than a line are broken by character). pub fn wrap(text: &str, size: f64, width: f64) -> Vec { + wrap_with(text, size, width, helvetica_width) +} + +/// [`wrap`], measuring with `measure` (for example [`cjk_width`]). +pub fn wrap_with(text: &str, size: f64, width: f64, measure: impl Fn(&str, f64) -> f64) -> Vec { let mut lines = Vec::new(); for para in text.split(['\n', '\r']) { let mut line = String::new(); for word in para.split(' ') { let candidate = if line.is_empty() { word.to_string() } else { format!("{line} {word}") }; - if helvetica_width(&candidate, size) <= width || line.is_empty() && helvetica_width(word, size) <= width { + if measure(&candidate, size) <= width || line.is_empty() && measure(word, size) <= width { line = candidate; continue; } @@ -55,7 +91,7 @@ pub fn wrap(text: &str, size: f64, width: f64) -> Vec { lines.push(std::mem::take(&mut line)); } for ch in word.chars() { - if !line.is_empty() && helvetica_width(&format!("{line}{ch}"), size) > width { + if !line.is_empty() && measure(&format!("{line}{ch}"), size) > width { lines.push(std::mem::take(&mut line)); } line.push(ch); @@ -87,6 +123,59 @@ pub fn win_ansi(s: &str) -> Vec { s.chars().map(|c| if c == '\t' { b' ' } else { win_ansi_byte(c).unwrap_or(b'?') }).collect() } +/// The character encoding of a predefined Unicode CMap (`UniJIS-UTF16-H`, `UniGB-UCS2-H`, …; +/// ISO 32000-2 §9.7.5.2), which maps Unicode text straight to codes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UnicodeCMap { + Ucs2, + Utf16, + Utf8, + Utf32, +} + +impl UnicodeCMap { + /// The encoding of the predefined CMap `name`, if it is a horizontal Unicode one. Vertical + /// (`-V`) CMaps are refused: field appearances are laid out horizontally, so their glyphs + /// would be drawn stacked down the field instead of across it. + pub fn from_name(name: &[u8]) -> Option { + let name = std::str::from_utf8(name).ok()?; + if !name.starts_with("Uni") || !name.ends_with("-H") { + return None; + } + name.split('-').find_map(|part| match part { + "UCS2" => Some(Self::Ucs2), + "UTF16" => Some(Self::Utf16), + "UTF8" => Some(Self::Utf8), + "UTF32" => Some(Self::Utf32), + _ => None, + }) + } + + /// Encode `s`; tabs become spaces, and characters UCS-2 can't hold become `?`. + pub fn encode(self, s: &str) -> Vec { + let mut out = Vec::with_capacity(s.len().saturating_mul(2)); + let mut units = [0u16; 2]; + let mut buf = [0u8; 4]; + for c in s.chars() { + let c = if c == '\t' { ' ' } else { c }; + match self { + Self::Ucs2 => { + let unit = u16::try_from(u32::from(c)).unwrap_or(u16::from(b'?')); + out.extend_from_slice(&unit.to_be_bytes()); + } + Self::Utf16 => { + for unit in c.encode_utf16(&mut units) { + out.extend_from_slice(&unit.to_be_bytes()); + } + } + Self::Utf8 => out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()), + Self::Utf32 => out.extend_from_slice(&u32::from(c).to_be_bytes()), + } + } + out + } +} + /// Bytes as a PDF literal string, `(` … `)`, with delimiters escaped. pub fn literal(bytes: &[u8]) -> Vec { let mut out = Vec::with_capacity(bytes.len() + 2); @@ -136,4 +225,26 @@ mod tests { assert_eq!(win_ansi("čć\u{81}\u{8d}\t"), b"???? "); assert_eq!(win_ansi_byte('\0'), None); } + + #[test] + fn unicode_cmaps_encode_cjk_text() { + assert_eq!(UnicodeCMap::from_name(b"UniJIS-UTF16-H"), Some(UnicodeCMap::Utf16)); + assert_eq!(UnicodeCMap::from_name(b"UniGB-UCS2-V"), None); + assert_eq!(UnicodeCMap::from_name(b"UniJIS-UCS2-HW-H"), Some(UnicodeCMap::Ucs2)); + assert_eq!(UnicodeCMap::from_name(b"UniKS-UTF8-H"), Some(UnicodeCMap::Utf8)); + assert_eq!(UnicodeCMap::from_name(b"UniJIS2004-UTF32-H"), Some(UnicodeCMap::Utf32)); + assert_eq!(UnicodeCMap::from_name(b"Identity-H"), None); + assert_eq!(UnicodeCMap::from_name(b"90ms-RKSJ-H"), None); + assert_eq!(UnicodeCMap::from_name(b"\xff\xfe"), None); + assert_eq!(UnicodeCMap::Utf16.encode("令A"), [0x4e, 0xe4, 0x00, 0x41]); + assert_eq!(UnicodeCMap::Utf16.encode("𠀋"), [0xd8, 0x40, 0xdc, 0x0b]); + assert_eq!(UnicodeCMap::Ucs2.encode("𠀋\t"), [0x00, b'?', 0x00, b' ']); + assert_eq!(UnicodeCMap::Utf8.encode("令"), "令".as_bytes()); + assert_eq!(UnicodeCMap::Utf32.encode("令"), [0, 0, 0x4e, 0xe4]); + assert_eq!(cjk_width("令和", 10.0), 20.0); + assert_eq!(cjk_width("ｱ", 10.0), 5.0); + assert_eq!(cjk_width("A", 10.0), helvetica_width("A", 10.0)); + let lines = wrap_with("日本語のテキストは文字単位で折り返されます", 10.0, 45.0, cjk_width); + assert!(lines.len() > 3 && lines.iter().all(|l| cjk_width(l, 10.0) <= 45.0), "{lines:?}"); + } } diff --git a/crates/forms/src/appearance.rs b/crates/forms/src/appearance.rs index fc4e36d21f..38050cfb88 100644 --- a/crates/forms/src/appearance.rs +++ b/crates/forms/src/appearance.rs @@ -5,12 +5,14 @@ //! marked-content block as viewers expect. Supported: single-line, multiline (wrapped), comb, //! password, quadding, auto font size (`0 Tf`), combo boxes and list boxes (selection shown). //! -//! The `/DA` font is used when the form's `/DR` defines it as a simple font; text is encoded in -//! WinAnsi. Otherwise (composite fonts, missing resources) Helvetica is used, so text is always -//! visible. Widths use the approximate Helvetica metrics of `pdfcraft-fonts`. +//! The `/DA` font is used when the form's `/DR` defines it as a simple font (text encoded in +//! WinAnsi) or as a composite font with a predefined Unicode CMap such as `UniJIS-UTF16-H`, as +//! Japanese forms use (text encoded in that CMap). Otherwise (other composite fonts, missing +//! resources) Helvetica is used, so text is always visible. Widths use the approximate metrics +//! of `pdfcraft-fonts` (Helvetica, or one em per full-width character in composite fonts). use pdfcraft_cos::{Dict, Document, Object, Stream}; -use pdfcraft_fonts::{helvetica_width, literal, win_ansi, wrap}; +use pdfcraft_fonts::{UnicodeCMap, cjk_width, helvetica_width, literal, win_ansi, wrap_with}; use crate::{Field, FieldKind, Widget, acroform, flags}; @@ -53,8 +55,32 @@ pub fn parse_da(da: &str) -> Da { out } -/// The font resource for `name` from the form's `/DR`, if it is a simple font we can encode for. -fn dr_font(doc: &Document, name: &str) -> Option { +/// How a field font's text is encoded and measured. +#[derive(Clone, Copy, Debug, PartialEq)] +enum FontText { + WinAnsi, + /// A composite font with a predefined Unicode CMap. + Unicode(UnicodeCMap), +} + +impl FontText { + fn encode(self, s: &str) -> Vec { + match self { + Self::WinAnsi => win_ansi(s), + Self::Unicode(cmap) => cmap.encode(s), + } + } + + fn width(self, s: &str, size: f64) -> f64 { + match self { + Self::WinAnsi => helvetica_width(s, size), + Self::Unicode(_) => cjk_width(s, size), + } + } +} + +/// The font resource for `name` from the form's `/DR`, if it is a font we can encode for. +fn dr_font(doc: &Document, name: &str) -> Option<(Object, FontText)> { let af = acroform(doc)?; let dr = doc.resolve(af.get(b"DR")?); let fonts = doc.resolve(dr.as_dict()?.get(b"Font")?); @@ -63,13 +89,17 @@ fn dr_font(doc: &Document, name: &str) -> Option { let fd = font.as_dict()?; match fd.name(b"Subtype") { Some(b"Type1" | b"TrueType" | b"MMType1") => {} + // Japanese, Chinese and Korean forms: a CID font addressed by Unicode (a non-embedded + // `/HeiseiMin-W3` with `/UniJIS-UCS2-H`, say). Embedded CMap streams and Identity-H need + // the font's own mapping, so those still fall back to Helvetica. + Some(b"Type0") => return Some((entry, FontText::Unicode(UnicodeCMap::from_name(fd.name(b"Encoding")?)?))), _ => return None, } // Symbolic fonts (ZapfDingbats, Symbol) can't show WinAnsi text. if matches!(fd.name(b"BaseFont"), Some(b"ZapfDingbats" | b"Symbol")) { return None; } - Some(entry) + Some((entry, FontText::WinAnsi)) } fn helvetica() -> Object { @@ -194,10 +224,11 @@ pub fn field_appearance_as(doc: &Document, f: &Field, w: &Widget, values: &[Stri let wd = wobj.as_dict().cloned().unwrap_or_default(); let (width, height) = ((w.rect[2] - w.rect[0]).max(1.0), (w.rect[3] - w.rect[1]).max(1.0)); let da = parse_da(wd.get(b"DA").and_then(|o| doc.resolve(o).as_string().map(|s| s.to_text())).as_deref().unwrap_or(&f.da)); - let (font_name, font_obj) = match dr_font(doc, &da.font) { - Some(o) => (da.font.clone(), o), - None => ("Helv".to_string(), helvetica()), + let (font_name, font_obj, enc) = match dr_font(doc, &da.font) { + Some((o, enc)) => (da.font.clone(), o, enc), + None => ("Helv".to_string(), helvetica(), FontText::WinAnsi), }; + let width_of = |text: &str, size: f64| enc.width(text, size); let (mut c, bw) = frame(doc, &wd, width, height); let pad = 2.0 + bw; let inner_w = (width - 2.0 * pad).max(1.0); @@ -205,11 +236,11 @@ pub fn field_appearance_as(doc: &Document, f: &Field, w: &Widget, values: &[Stri let mut body: Vec = Vec::new(); let show = |body: &mut Vec, x: f64, y: f64, text: &str| { body.extend(format!("1 0 0 1 {} {} Tm ", n(x), n(y)).bytes()); - body.extend(literal(&win_ansi(text))); + body.extend(literal(&enc.encode(text))); body.extend_from_slice(b" Tj\n"); }; let x_for = |text: &str, size: f64| -> f64 { - let tw = helvetica_width(text, size); + let tw = width_of(text, size); match q { 1 => pad + (inner_w - tw) / 2.0, 2 => width - pad - tw, @@ -249,12 +280,12 @@ pub fn field_appearance_as(doc: &Document, f: &Field, w: &Widget, values: &[Stri if size == 0.0 { // Auto size: the largest size (≤ 12) whose wrapped lines fit the height. size = 12.0; - while size > 4.0 && wrap(&text, size, inner_w).len() as f64 * size * 1.15 > height - 2.0 * pad { + while size > 4.0 && wrap_with(&text, size, inner_w, width_of).len() as f64 * size * 1.15 > height - 2.0 * pad { size -= 0.5; } } let mut y = height - pad - size * 0.85; - for line in wrap(&text, size, inner_w) { + for line in wrap_with(&text, size, inner_w, width_of) { if y < -size { break; } @@ -264,7 +295,7 @@ pub fn field_appearance_as(doc: &Document, f: &Field, w: &Widget, values: &[Stri } else { if size == 0.0 { size = ((height - 2.0 * pad) / 1.15).clamp(4.0, 12.0); - let tw = helvetica_width(&text, size); + let tw = width_of(&text, size); if tw > inner_w && !comb { size = (size * inner_w / tw).max(4.0); } @@ -276,7 +307,7 @@ pub fn field_appearance_as(doc: &Document, f: &Field, w: &Widget, values: &[Stri let cell = width / cells as f64; for (i, ch) in text.chars().take(cells).enumerate() { let s = ch.to_string(); - show(&mut body, cell * i as f64 + (cell - helvetica_width(&s, size)) / 2.0, y, &s); + show(&mut body, cell * i as f64 + (cell - width_of(&s, size)) / 2.0, y, &s); } } else { show(&mut body, x_for(&text, size), y, &text); diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 8e846edb72..74e4b0803b 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -159,6 +159,55 @@ fn text_fields_get_new_appearances() { assert!(raw.windows(8).any(|x| x == b"(Z\xfcrich)")); } +#[test] +fn japanese_choices_use_the_unicode_cid_font() { + // A choice field whose /DA names a non-embedded CID font with a predefined Unicode CMap + // (UniJIS-UTF16-H). Selecting 令 must draw it in that font, not "?" in Helvetica. + let objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [8 0 R 9 0 R 10 0 R] >>".into(), + "<< /Fields [8 0 R 9 0 R 10 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R /HeiseiMin-W3 6 0 R /Emb 7 0 R >> >> >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /Type /Font /Subtype /Type0 /BaseFont /HeiseiMin-W3 /Encoding /UniJIS-UTF16-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType0 /BaseFont /HeiseiMin-W3 /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>] >>".into(), + "<< /Type /Font /Subtype /Type0 /BaseFont /ABCDEF+Subset /Encoding /Identity-H /DescendantFonts [] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Ch /T (era) /Ff 131072 /DA (/HeiseiMin-W3 10 Tf 0 g) /Q 1 /Opt [ ] /V /Rect [50 700 80 715] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (address) /Ff 4096 /DA (/HeiseiMin-W3 10 Tf 0 g) /Rect [50 600 100 680] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (subset) /DA (/Emb 10 Tf 0 g) /Rect [50 500 100 520] /P 3 0 R >>".into(), + ]; + let mut doc = document(&objs); + assert_eq!(field(&fields(&doc), "era").options.get(2), Some(&("令".to_string(), "令".to_string()))); + set_value(&mut doc, "era", &FieldValue::Text("令".into())).unwrap(); + set_value(&mut doc, "address", &FieldValue::Text("日本語のテキスト入力欄です".into())).unwrap(); + set_value(&mut doc, "subset", &FieldValue::Text("令和".into())).unwrap(); + let doc = reopen(&doc); + let all = fields(&doc); + assert_eq!(field(&all, "era").value, ["令"]); + let raw = |name: &str| { + let w = &field(&all, name).widgets[0]; + let n = doc.get(w.obj).as_dict().unwrap().get(b"AP").unwrap().as_dict().unwrap().reference(b"N").unwrap(); + let Object::Stream(s) = &*doc.get(n) else { panic!() }; + let fonts = doc.resolve(s.dict.get(b"Resources").unwrap()).as_dict().unwrap().get(b"Font").map(|f| doc.resolve(f)).unwrap(); + let fonts: Vec = fonts.as_dict().unwrap().iter().map(|(k, _)| String::from_utf8_lossy(k).into_owned()).collect(); + (s.decoded().unwrap(), fonts) + }; + let utf16 = |t: &str| t.encode_utf16().flat_map(u16::to_be_bytes).collect::>(); + let (era, era_fonts) = raw("era"); + assert!(era.windows(4).any(|x| x == b"(N\xe4)"), "令 as UTF-16BE: {}", String::from_utf8_lossy(&era)); + assert!(String::from_utf8_lossy(&era).contains("/HeiseiMin-W3 10 Tf")); + assert!(!era.windows(3).any(|x| x == b"(?)"), "no WinAnsi fallback"); + assert_eq!(era_fonts, ["HeiseiMin-W3"]); + // Multiline Japanese wraps by character at about one em each (50 pt wide, 10 pt type). + let (addr, _) = raw("address"); + let addr_text = String::from_utf8_lossy(&addr); + assert!(addr_text.matches(" Tj").count() >= 4, "{addr_text}"); + assert!(addr.windows(4).any(|x| x == utf16("日本").as_slice())); + // An Identity-H subset can't be addressed by Unicode: Helvetica, as before. + let (sub, sub_fonts) = raw("subset"); + assert!(String::from_utf8_lossy(&sub).contains("/Helv "), "{}", String::from_utf8_lossy(&sub)); + assert_eq!(sub_fonts, ["Helv"]); +} + #[test] fn check_boxes_and_radios_switch_states() { let mut doc = fixture();