diff --git a/ROADMAP.md b/ROADMAP.md index 44a26604d8..b8fc5d6956 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -141,6 +141,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-12 (M2, installed CJK fonts):** A CID font of an Adobe CJK collection (Japan1, GB1, CNS1, Korea1) that a PDF doesn't embed, such as the Kozuka Gothic of an Acrobat text box or the SongtiTC of a Pages export, drew nothing unless craft-fonts had a Japanese face. After craft-fonts, it now draws with a face already installed on the machine (PingFang, Heiti, Microsoft JhengHei / YaHei, Malgun and the like; read at runtime, never bundled; Noto CJK and Source Han excluded; `PDFCRAFT_SYSTEM_FONTS=0` turns it off). The face of a collection with several regions (PingFang SC, TC) is chosen by region. Checked on macOS with a synthetic Traditional Chinese, Simplified Chinese and Korean page; the Windows and Linux font lists are untested, and a machine with none of the listed faces still draws nothing. - **2026-10-12 (M1, #1339):** An incremental save of an unchanged classic-table file used to append `xref` and then `trailer` with no subsection, which qpdf reports as a damaged file. The section now has a zero-length subsection, so no object entry changes and a trailer-only edit is still written. Estimates unchanged. - **2026-10-12 (M12, auto-tag preview):** The engine and `accessibility_autotag_preview` propose paragraph and large-heading roles for bounded text blocks on one page, with locations and reasons. Every proposal needs review; images remain unclassified. No PDF tags are written yet. Synthetic tool tests cover normal text, a larger heading, bounds and unchanged source bytes. Estimates unchanged. - **2026-10-12 (M10, editing OCR text on scans):** Editing a recognised word on a scanned page now changes what the page shows. Before, the rewrite stayed invisible OCR text, so the scan still showed the old word; on a form, the whole OCR layer also grouped into one paragraph that rewrapped when one word changed. OCR words now group into phrases (a table cell or a tab stop splits them, and lines stay apart); an edit draws the phrase visibly in the scan's sampled ink colour over a cover in the sampled paper colour, keeping each word's width and growing rightwards. The editor covers the scanned word while typing. OCR words are now sized from their letters (a capitals-only word's box spans the cap height, not the full ascender-descender span), so recognised and rewritten text is as tall as the scanned word and sits on its baseline. Five engine tests on synthetic scans and a kittest UI test; `ocr.editable-text-output` is partial. diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 28202a38b4..71467dfeb2 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -13,6 +13,8 @@ pdfcraft-cos = { workspace = true } pdfcraft-fonts = { workspace = true } hayro = { workspace = true } hayro-syntax = { workspace = true } +# Picks the face of an installed font collection (the version pdfcraft-fonts already uses). +skrifa = "0.44" lopdf = { workspace = true } kurbo = { workspace = true } thiserror = { workspace = true } @@ -27,7 +29,5 @@ bytemuck = { version = "1" } getrandom = { version = "0.4", features = ["wasm_js"] } [dev-dependencies] -# Checks the Japanese substitute's character coverage; the version pdfcraft-fonts already uses. -skrifa = "0.44" # Builds the decompression-bomb streams in the raster tests. flate2 = { workspace = true } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index d2ee5712c4..90bc64b086 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -13,6 +13,7 @@ mod pixels; mod raster; mod reading; mod structure; +mod system_font; pub mod text; pub use inspect::{ diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 0e64ac241e..fe469958ae 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -73,7 +73,7 @@ impl RenderConfig { InterpreterSettings { ocg_overrides: self.layers.clone(), hide_comments: self.hide_comments, - font_resolver: Arc::new(move |query| japanese_fallback(query).or_else(|| standard(query))), + font_resolver: Arc::new(move |query| japanese_fallback(query).or_else(|| system_fallback(query)).or_else(|| standard(query))), ..InterpreterSettings::default() } } @@ -106,6 +106,14 @@ fn japanese_fallback(query: &FontQuery) -> Option<(FontData, u32)> { Some((Arc::new(face.bytes), 0)) } +/// An installed face for a CID font of an Adobe CJK collection that the PDF doesn't embed and +/// craft-fonts has no face for (Traditional and Simplified Chinese, Korean, and Japanese in a +/// build without craft-fonts). `None` for every other font. +fn system_fallback(query: &FontQuery) -> Option<(FontData, u32)> { + let FontQuery::Fallback(f) = query else { return None }; + crate::system_font::fallback(f) +} + #[derive(Debug, PartialEq)] enum JapaneseFace { Mincho, @@ -5515,6 +5523,57 @@ trailer << /Root 1 0 R >> } } + /// Chinese and Korean CID fonts that aren't embedded (Adobe-CNS1 `MSung-Light`, Adobe-GB1 + /// `STSong-Light`, Adobe-Korea1 `HYGoThic-Medium`; Acrobat also writes Japanese `KozGoPr6N` + /// text boxes without the font) drew nothing, because craft-fonts has only Japanese faces. + /// They now draw in a face already installed on the machine, read at runtime. A machine + /// without such a face draws nothing, as before, so each check is skipped there. + #[test] + fn non_embedded_chinese_and_korean_cid_fonts_draw_with_an_installed_face() { + use hayro::hayro_interpret::font::FallbackFontQuery; + use hayro::hayro_interpret::hayro_cmap::CharacterCollection; + // (base font, collection, ordering, encoding, UCS-2 code units of the text) + let cases = [ + ("MSung-Light", CidFamily::AdobeCNS1, "CNS1", "UniCNS-UCS2-H", "7E419AD45B57"), // 繁體字 + ("STSong-Light", CidFamily::AdobeGB1, "GB1", "UniGB-UCS2-H", "72698D448DF7"), // 物资跷 (common Simplified) + ("HYGoThic-Medium", CidFamily::AdobeKorea1, "Korea1", "UniKS-UCS2-H", "D55CAE00"), // 한글 + ]; + for (base_font, family, ordering, encoding, code) in cases { + let query = FontQuery::Fallback(FallbackFontQuery { + post_script_name: Some(base_font.into()), + character_collection: Some(CharacterCollection { family, supplement: 2 }), + ..FallbackFontQuery::default() + }); + if super::system_fallback(&query).is_none() { + eprintln!("no installed face for {base_font} (or PDFCRAFT_SYSTEM_FONTS=0): skipped"); + continue; + } + let content = format!("BT /F1 40 Tf 5 15 Td <{code}> Tj ET"); + let pdf = format!( + "%PDF-1.7 +1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj +2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj +3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 60] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj +4 0 obj << /Length {} >> stream +{content} +endstream endobj +5 0 obj << /Type /Font /Subtype /Type0 /BaseFont /{base_font} /Encoding /{encoding} /DescendantFonts [6 0 R] >> endobj +6 0 obj << /Type /Font /Subtype /CIDFontType0 /BaseFont /{base_font} /CIDSystemInfo << /Registry (Adobe) /Ordering ({ordering}) /Supplement 2 >> + /FontDescriptor 7 0 R /DW 1000 >> endobj +7 0 obj << /Type /FontDescriptor /FontName /{base_font} /Flags 6 /FontBBox [0 -141 1000 859] /ItalicAngle 0 /Ascent 859 /Descent -141 /CapHeight 700 /StemV 80 >> endobj +trailer << /Root 1 0 R >> +%%EOF", + content.len() + ); + let mut r = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let p = r.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 1.0, tag: 0, ..Default::default() }); + assert!(p.error.is_none(), "{base_font}: {:?}", p.error); + let inked = p.rgba.as_chunks::<4>().0.iter().filter(|c| c[0] < 128).count(); + // Two to four glyphs at 40 pt cover several hundred dark pixels; a blank run has none. + assert!(inked > 300, "{base_font}: the text is drawn ({inked} dark pixels)"); + } + } + /// Half-width katakana are common in Japanese documents set in a non-embedded Mincho font /// such as HeiseiMin-W3. The Mincho substitute must have those glyphs: one that lacks them /// (Shippori Mincho) draws them as .notdef. diff --git a/crates/render/src/system_font.rs b/crates/render/src/system_font.rs new file mode 100644 index 0000000000..d2c98bb7cd --- /dev/null +++ b/crates/render/src/system_font.rs @@ -0,0 +1,248 @@ +//! Installed CJK faces for CID fonts a PDF doesn't embed. +//! +//! Acrobat and other tools often write Chinese, Japanese or Korean text with one of the Adobe +//! collections (Adobe-Japan1, GB1, CNS1, Korea1) and a font name such as `KozGoPr6N-Medium` or +//! `MSung-Light` without embedding the font. hayro maps those CIDs to Unicode itself, but it needs +//! a face to draw them with; without one the text is blank. The craft-fonts Japanese faces come +//! first (see `japanese_fallback` in `raster.rs`); this module is the last resort: one face +//! already installed on this machine. It is read at runtime and never embedded or shipped +//! (AGENTS.md §1.4), Noto CJK and Source Han are left out on purpose (§1.1), and +//! `PDFCRAFT_SYSTEM_FONTS=0` turns it off, as for the interface font. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; + +use hayro::hayro_interpret::font::{FallbackFontQuery, FontData}; +use hayro::hayro_interpret::hayro_cmap::CidFamily; +use skrifa::MetadataProvider as _; +use skrifa::string::StringId; + +/// Larger files are not read: a font path is still untrusted input. CJK system faces are big +/// collections (PingFang is tens of megabytes). +const MAX_BYTES: u64 = 96 << 20; +/// Faces tried in a collection (`.ttc`). +const MAX_FACES: u32 = 32; + +/// The writing system a character collection is for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum Script { + Japanese, + SimplifiedChinese, + TraditionalChinese, + Korean, +} + +impl Script { + fn of(family: &CidFamily) -> Option { + match family { + CidFamily::AdobeJapan1 => Some(Self::Japanese), + CidFamily::AdobeGB1 => Some(Self::SimplifiedChinese), + CidFamily::AdobeCNS1 => Some(Self::TraditionalChinese), + CidFamily::AdobeKorea1 => Some(Self::Korean), + CidFamily::AdobeIdentity | CidFamily::Custom { .. } => None, + } + } + + /// A character the face must have to be worth using. Each is common and not shared by the + /// other scripts' faces: 物 (Japanese; also Chinese, so it only rules out Latin-only faces), + /// 欢 (Simplified only), 說 (Traditional only), 한 (Hangul). + fn probe(self) -> char { + match self { + Self::Japanese => '\u{7269}', + Self::SimplifiedChinese => '\u{6B22}', + Self::TraditionalChinese => '\u{8AAA}', + Self::Korean => '\u{D55C}', + } + } +} + +/// One installed file and the words (in a face's family name) that pick the face for the right +/// region out of a collection that holds several (PingFang SC, TC and HK share one file). +struct Candidate { + path: PathBuf, + region: &'static [&'static str], +} + +/// The installed face to draw a CID font of `query`'s collection with, as `(data, face index)`. +/// `None` when it is turned off, the collection isn't one of the four, or nothing fits. +pub(crate) fn fallback(query: &FallbackFontQuery) -> Option<(FontData, u32)> { + if cfg!(target_arch = "wasm32") || std::env::var_os("PDFCRAFT_SYSTEM_FONTS").is_some_and(|v| v == "0") { + return None; + } + let script = Script::of(&query.character_collection.as_ref()?.family)?; + let bold = query.is_bold || query.font_weight >= 600; + /// A face file and the index of the chosen face in it, or `None` when nothing fits. + type Face = Option<(Arc>, u32)>; + static CACHE: Mutex>> = Mutex::new(None); + let mut cache = CACHE.lock().unwrap_or_else(PoisonError::into_inner); + let found = cache.get_or_insert_with(HashMap::new).entry((script, bold)).or_insert_with(|| { + candidates(script) + .iter() + .find_map(|c| read(c, script.probe(), bold)) + .inspect(|(_, index)| log::info!("CJK font fallback for {script:?}: face {index}")) + }); + found.as_ref().map(|(bytes, index)| (Arc::clone(bytes) as FontData, *index)) +} + +fn read(candidate: &Candidate, probe: char, bold: bool) -> Option<(Arc>, u32)> { + let meta = std::fs::metadata(&candidate.path).ok()?; + if !meta.is_file() || meta.len() > MAX_BYTES { + return None; + } + let bytes = std::fs::read(&candidate.path).ok()?; + let index = pick_face(&bytes, probe, candidate.region, bold)?; + Some((Arc::new(bytes), index)) +} + +/// The best face of the file that parses and maps `probe`: the region's face when the file has +/// one, then the regular (or, for bold text, the semibold or bold) weight, then the first. +fn pick_face(bytes: &[u8], probe: char, region: &[&str], bold: bool) -> Option { + let mut best: Option<(u32, u32)> = None; + for index in 0..MAX_FACES { + let Ok(font) = skrifa::FontRef::from_index(bytes, index) else { continue }; + if font.charmap().map(probe).is_none() { + continue; + } + let names = |id: StringId| font.localized_strings(id).map(|s| s.chars().collect::()).collect::>(); + let family = names(StringId::FAMILY_NAME).join(" "); + let style = names(StringId::SUBFAMILY_NAME).join(" ").to_ascii_lowercase(); + let mut score = 1; + if region.iter().any(|r| family.split_whitespace().any(|w| w == *r)) { + score += 4; + } + let weighty = style.contains("bold") || style.contains("semibold") || style.contains("heavy"); + let plain = style.is_empty() || style.contains("regular") || style.contains("normal") || style.contains("medium") && !bold; + if (bold && weighty) || (!bold && plain && !weighty) { + score += 2; + } + if best.is_none_or(|(_, s)| score > s) { + best = Some((index, score)); + } + } + best.map(|(index, _)| index) +} + +fn candidates(script: Script) -> Vec { + let region: &'static [&'static str] = match script { + Script::Japanese => &["JP", "Japanese"], + Script::SimplifiedChinese => &["SC", "GB"], + Script::TraditionalChinese => &["TC", "HK", "CNS"], + Script::Korean => &["KR", "Korean"], + }; + let files: Vec = if cfg!(windows) { + let dir = windows_fonts(); + let names: &[&str] = match script { + Script::Japanese => &["YuGothR.ttc", "meiryo.ttc", "msgothic.ttc"], + Script::SimplifiedChinese => &["msyh.ttc", "simsun.ttc", "msjh.ttc"], + Script::TraditionalChinese => &["msjh.ttc", "mingliu.ttc", "msyh.ttc"], + Script::Korean => &["malgun.ttf", "gulim.ttc"], + }; + names.iter().map(|f| dir.join(f)).collect() + } else if cfg!(target_os = "macos") { + let names: &[&str] = match script { + Script::Japanese => &["ヒラギノ角ゴシック W3.ttc", "Hiragino Sans GB.ttc", "PingFang.ttc"], + Script::SimplifiedChinese => &["PingFang.ttc", "Hiragino Sans GB.ttc", "STHeiti Light.ttc"], + Script::TraditionalChinese => &["PingFang.ttc", "STHeiti Light.ttc", "Hiragino Sans GB.ttc"], + Script::Korean => &["AppleSDGothicNeo.ttc", "AppleGothic.ttf", "PingFang.ttc"], + }; + names.iter().map(|f| Path::new("/System/Library/Fonts").join(f)).collect() + } else { + let names: &[&str] = match script { + Script::Korean => { + &["truetype/nanum/NanumGothic.ttf", "nanum/NanumGothic.ttf", "truetype/baekmuk/gulim.ttf", "truetype/unfonts-core/UnDotum.ttf"] + } + Script::Japanese => &[ + "truetype/fonts-japanese-gothic.ttf", + "ipa-gothic/ipag.ttf", + "truetype/takao-gothic/TakaoGothic.ttf", + "truetype/vlgothic/VL-Gothic-Regular.ttf", + ], + Script::SimplifiedChinese | Script::TraditionalChinese => &[ + "truetype/wqy/wqy-microhei.ttc", + "wenquanyi/wqy-microhei/wqy-microhei.ttc", + "wqy-microhei/wqy-microhei.ttc", + "truetype/wqy/wqy-zenhei.ttc", + "wenquanyi/wqy-zenhei/wqy-zenhei.ttc", + "wqy-zenhei/wqy-zenhei.ttc", + "truetype/droid/DroidSansFallbackFull.ttf", + "google-droid-sans-fonts/DroidSansFallbackFull.ttf", + "truetype/arphic/uming.ttc", + "arphic-uming/uming.ttc", + ], + }; + ["/usr/share/fonts", "/usr/local/share/fonts"].iter().flat_map(|dir| names.iter().map(move |f| Path::new(dir).join(f))).collect() + }; + files.into_iter().map(|path| Candidate { path, region }).collect() +} + +fn windows_fonts() -> PathBuf { + let dir = std::env::var_os("WINDIR").or_else(|| std::env::var_os("SystemRoot")).map_or_else(|| PathBuf::from(r"C:\Windows"), PathBuf::from); + dir.join("Fonts") +} + +#[cfg(test)] +mod tests { + use super::*; + + const SCRIPTS: [Script; 4] = [Script::Japanese, Script::SimplifiedChinese, Script::TraditionalChinese, Script::Korean]; + + #[test] + fn broken_and_missing_files_are_skipped() { + assert_eq!(pick_face(b"", '\u{7269}', &[], false), None); + assert_eq!(pick_face(b"not a font at all", '\u{7269}', &[], false), None); + assert_eq!(pick_face(&[0u8; 4096], '\u{7269}', &[], false), None); + let missing = Candidate { path: PathBuf::from("definitely/not/here.ttc"), region: &[] }; + assert!(read(&missing, '\u{7269}', false).is_none()); + // A directory is not a font. + let dir = Candidate { path: std::env::temp_dir(), region: &[] }; + assert!(read(&dir, '\u{7269}', false).is_none()); + } + + #[test] + fn a_latin_face_is_rejected_for_every_script() { + let inter = include_bytes!("../../../assets/fonts/Inter-Regular.ttf"); + for script in SCRIPTS { + assert_eq!(pick_face(inter, script.probe(), &[], false), None, "{script:?}"); + } + assert_eq!(pick_face(inter, 'A', &[], false), Some(0)); + } + + #[test] + fn candidates_are_absolute_font_files_without_adobe_designs() { + for script in SCRIPTS { + let list = candidates(script); + assert!(!list.is_empty(), "{script:?}"); + for c in list { + assert!(c.path.is_absolute(), "{}", c.path.display()); + assert!(c.path.extension().is_some_and(|e| e == "ttf" || e == "ttc"), "{}", c.path.display()); + let name = c.path.to_string_lossy().to_lowercase(); + assert!(!["notosanscjk", "notoserifcjk", "sourcehan", "source-han"].iter().any(|b| name.contains(b)), "{name}"); + } + } + } + + #[test] + fn only_the_four_adobe_collections_have_a_script() { + assert_eq!(Script::of(&CidFamily::AdobeCNS1), Some(Script::TraditionalChinese)); + assert_eq!(Script::of(&CidFamily::AdobeGB1), Some(Script::SimplifiedChinese)); + assert_eq!(Script::of(&CidFamily::AdobeKorea1), Some(Script::Korean)); + assert_eq!(Script::of(&CidFamily::AdobeJapan1), Some(Script::Japanese)); + assert_eq!(Script::of(&CidFamily::AdobeIdentity), None); + } + + /// On a Mac the face for Traditional Chinese is the TC one of PingFang, not the first of the + /// collection (which is another region's), and it maps a character only Traditional has. + #[cfg(target_os = "macos")] + #[test] + fn macos_picks_the_regional_pingfang_face() { + let path = Path::new("/System/Library/Fonts/PingFang.ttc"); + let Ok(bytes) = std::fs::read(path) else { return }; + for (script, region) in [(Script::TraditionalChinese, "TC"), (Script::SimplifiedChinese, "SC")] { + let index = pick_face(&bytes, script.probe(), &[region], false).expect("PingFang covers it"); + let font = skrifa::FontRef::from_index(&bytes, index).unwrap(); + let family: String = font.localized_strings(StringId::FAMILY_NAME).next().unwrap().chars().collect(); + assert!(family.contains(region), "{script:?} got {family:?}"); + } + } +}