Merge pull request #38 from justinjohn0306/fix/thai-hebrew-font-fallback

Fonts: render Thai and Hebrew (and other non-Latin scripts) instead of tofu (#32)

crates/ui-egui/src/theme.rs

@@ -292,6 +292,9 @@ pub fn font_definitions() -> FontDefinitions {
     let mut fonts = FontDefinitions::default();
     let base_prop: Vec<String> = fonts.families.get(&FontFamily::Proportional).cloned().unwrap_or_default();
     let base_mono: Vec<String> = fonts.families.get(&FontFamily::Monospace).cloned().unwrap_or_default();
+    // The non-Latin fallback faces have to sit at the end of each chain: a Latin font owns Latin,
+    // and epaint picks the first face that has the glyph, so they only catch their own scripts.
+    let fallback = load_fallback(&mut fonts);
     for role in [UI, UI_BOLD, CELL, CELL_BOLD, CELL_ITALIC, CELL_BOLD_ITALIC, SERIF, MONO] {
         let paths = candidates(role);
         let refs: Vec<(&str, u32)> = paths.iter().map(|(p, i)| (p.as_str(), *i)).collect();
@@ -306,6 +309,7 @@ pub fn font_definitions() -> FontDefinitions {
             chain.push(format!("sys-{CELL}"));
         }
         chain.extend(if role == MONO { base_mono.clone() } else { base_prop.clone() });
+        chain.extend(fallback.iter().cloned());
         chain.retain(|k| fonts.font_data.contains_key(k));
         fonts.families.insert(FontFamily::Name(role.into()), chain);
     }
@@ -316,6 +320,156 @@ pub fn font_definitions() -> FontDefinitions {
     fonts
 }
 
+/// Loads the system non-Latin fallback faces (one file per script family) and returns their keys so
+/// every family can carry them as fallback. egui's bundled fonts have no coverage beyond Latin, so
+/// without this a Thai, Hebrew, Japanese or Chinese workbook shows tofu boxes even though the text
+/// was read correctly. With a face installed those scripts get real glyphs instead of tofu; nothing
+/// is bundled, and a machine without the face keeps the old behaviour.
+#[cfg(not(target_arch = "wasm32"))]
+fn load_fallback(fonts: &mut FontDefinitions) -> Vec<String> {
+    let mut keys = Vec::new();
+    // Files already loaded as a fallback face. A file serves at most one key: the region-merged
+    // Noto Sans CJK .ttc found for Han also carries Hangul (and Tahoma carries both Thai and
+    // Hebrew), so loading it again for a later script would only duplicate tens of MB in memory.
+    let mut loaded: Vec<String> = Vec::new();
+    for (key, cands) in fallback_faces() {
+        for (path, index) in &cands {
+            if loaded.contains(path) {
+                break; // a face loaded for an earlier script already covers this one
+            }
+            if let Some(fd) = try_load(&[(path.as_str(), *index)]) {
+                fonts.font_data.insert(key.to_string(), Arc::new(fd));
+                keys.push(key.to_string());
+                loaded.push(path.clone());
+                break;
+            }
+        }
+    }
+    keys
+}
+
+#[cfg(target_arch = "wasm32")]
+fn load_fallback(_fonts: &mut FontDefinitions) -> Vec<String> {
+    Vec::new()
+}
+
+/// Every fallback face we look for, in chain order: `(key, candidate files)`. One file per script
+/// family is enough — epaint picks the first face with the glyph, and a script's glyphs live in one
+/// file. Order only matters between faces that both cover a glyph (e.g. Japanese vs Chinese kanji).
+#[cfg(not(target_arch = "wasm32"))]
+fn fallback_faces() -> Vec<(&'static str, Vec<(String, u32)>)> {
+    vec![("sys-cjk-han", cjk_han()), ("sys-cjk-hangul", cjk_hangul()), ("sys-thai", thai()), ("sys-hebrew", hebrew())]
+}
+
+/// Linux directories that hold CJK fonts across distributions: Debian/Ubuntu (`opentype/noto`,
+/// `truetype/wqy`), Fedora (`google-noto-cjk`), Arch (`noto-cjk`, `wenquanyi`), plus local and
+/// flat layouts.
+#[cfg(not(target_arch = "wasm32"))]
+const LINUX_CJK_DIRS: [&str; 10] = [
+    "/usr/share/fonts/opentype/noto",
+    "/usr/share/fonts/google-noto-cjk",
+    "/usr/share/fonts/noto-cjk",
+    "/usr/share/fonts/truetype/noto",
+    "/usr/share/fonts/noto",
+    "/usr/share/fonts/truetype",
+    "/usr/share/fonts/TTF",
+    "/usr/share/fonts",
+    "/usr/local/share/fonts",
+    "/usr/local/share/fonts/noto-cjk",
+];
+
+/// Han + Kana faces, in fallback order. A Japanese face comes first (kana and JIS kanji render
+/// with Japanese glyph shapes); a Chinese face follows to widen coverage. Every CJK face carries
+/// kana, so any of them renders Japanese; the order only decides kanji glyph style.
+///
+/// The region-merged `NotoSansCJK-Regular.ttc` holds one face per locale (JP 0, KR 1, SC 2, TC 3,
+/// HK 4) with the same glyph coverage; index 0 picks the Japanese glyph shapes.
+#[cfg(not(target_arch = "wasm32"))]
+fn cjk_han() -> Vec<(String, u32)> {
+    let mac = "/System/Library/Fonts";
+    let win = std::env::var("WINDIR").map(|w| format!("{w}\\Fonts")).unwrap_or_else(|_| "C:\\Windows\\Fonts".into());
+    let mut v: Vec<(String, u32)> = Vec::new();
+    v.push((format!("{win}\\YuGothR.ttc"), 0)); // Yu Gothic (Japanese)
+    v.push((format!("{win}\\msgothic.ttc"), 0)); // MS Gothic (Japanese)
+    v.push((format!("{win}\\msyh.ttc"), 0)); // Microsoft YaHei (Chinese)
+    v.push((format!("{win}\\simsun.ttc"), 0)); // SimSun (Chinese, older installs)
+    v.push((format!("{mac}/ヒラギノ角ゴシック W3.ttc"), 0)); // Hiragino Sans (Japanese)
+    v.push((format!("{mac}/Hiragino Sans GB.ttc"), 0)); // ships on every recent macOS
+    v.push((format!("{mac}/PingFang.ttc"), 0)); // older macOS only; now a downloadable asset
+    for d in LINUX_CJK_DIRS {
+        v.push((format!("{d}/NotoSansCJK-Regular.ttc"), 0));
+        v.push((format!("{d}/NotoSansCJKjp-Regular.otf"), 0));
+        v.push((format!("{d}/NotoSansCJKsc-Regular.otf"), 0));
+    }
+    for d in LINUX_CJK_DIRS {
+        v.push((format!("{d}/wqy/wqy-microhei.ttc"), 0));
+        v.push((format!("{d}/wenquanyi/wqy-microhei.ttc"), 0));
+        v.push((format!("{d}/wqy-microhei.ttc"), 0));
+    }
+    v
+}
+
+/// Hangul faces (Korean). The Han faces above that come from the region-merged Noto Sans CJK file
+/// already carry Hangul; [`load_fallback`] then skips this list rather than load the file twice.
+#[cfg(not(target_arch = "wasm32"))]
+fn cjk_hangul() -> Vec<(String, u32)> {
+    let mac = "/System/Library/Fonts";
+    let win = std::env::var("WINDIR").map(|w| format!("{w}\\Fonts")).unwrap_or_else(|_| "C:\\Windows\\Fonts".into());
+    let mut v: Vec<(String, u32)> = Vec::new();
+    v.push((format!("{win}\\malgun.ttf"), 0)); // Malgun Gothic
+    v.push((format!("{mac}/AppleSDGothicNeo.ttc"), 0));
+    for d in LINUX_CJK_DIRS {
+        v.push((format!("{d}/NotoSansCJKkr-Regular.otf"), 0));
+        v.push((format!("{d}/NotoSansCJK-Regular.ttc"), 1)); // KR face
+    }
+    v
+}
+
+/// Thai faces. Windows ships Leelawadee UI and Tahoma/Microsoft Sans Serif (both carry Thai);
+/// macOS has Thonburi; Linux usually has Noto Sans Thai or Garuda.
+#[cfg(not(target_arch = "wasm32"))]
+fn thai() -> Vec<(String, u32)> {
+    let mac_sys = "/System/Library/Fonts";
+    let mac_sup = "/System/Library/Fonts/Supplemental";
+    let win = std::env::var("WINDIR").map(|w| format!("{w}\\Fonts")).unwrap_or_else(|_| "C:\\Windows\\Fonts".into());
+    let lin = ["/usr/share/fonts/truetype/noto", "/usr/share/fonts/opentype/noto", "/usr/share/fonts/truetype", "/usr/share/fonts"];
+    let mut v: Vec<(String, u32)> = Vec::new();
+    v.push((format!("{win}\\LeelawUI.ttf"), 0)); // Leelawadee UI
+    v.push((format!("{win}\\tahoma.ttf"), 0)); // Tahoma
+    v.push((format!("{win}\\micross.ttf"), 0)); // Microsoft Sans Serif
+    v.push((format!("{mac_sup}/Thonburi.ttc"), 0));
+    v.push((format!("{mac_sys}/SukhumvitSet.ttc"), 0));
+    for d in lin {
+        v.push((format!("{d}/NotoSansThai-Regular.ttf"), 0));
+        v.push((format!("{d}/tlwg/Garuda.ttf"), 0));
+        v.push((format!("{d}/garuda/Garuda.ttf"), 0));
+    }
+    v
+}
+
+/// Hebrew faces. Windows has David and FrankRuehl (and Tahoma/Courier New carry Hebrew); macOS has
+/// Arial Hebrew and David; Linux usually has Noto Sans Hebrew or DejaVu Sans.
+#[cfg(not(target_arch = "wasm32"))]
+fn hebrew() -> Vec<(String, u32)> {
+    let mac_sys = "/System/Library/Fonts";
+    let mac_sup = "/System/Library/Fonts/Supplemental";
+    let win = std::env::var("WINDIR").map(|w| format!("{w}\\Fonts")).unwrap_or_else(|_| "C:\\Windows\\Fonts".into());
+    let lin = ["/usr/share/fonts/truetype/noto", "/usr/share/fonts/opentype/noto", "/usr/share/fonts/truetype", "/usr/share/fonts"];
+    let mut v: Vec<(String, u32)> = Vec::new();
+    v.push((format!("{win}\\david.ttf"), 0)); // David
+    v.push((format!("{win}\\frank.ttf"), 0)); // FrankRuehl
+    v.push((format!("{win}\\tahoma.ttf"), 0)); // Tahoma
+    v.push((format!("{mac_sup}/Arial Hebrew.ttf"), 0));
+    v.push((format!("{mac_sup}/David.ttf"), 0));
+    v.push((format!("{mac_sys}/Lucida Grande.ttc"), 0));
+    for d in lin {
+        v.push((format!("{d}/NotoSansHebrew-Regular.ttf"), 0));
+        v.push((format!("{d}/dejavu/DejaVuSans.ttf"), 0));
+        v.push((format!("{d}/freefont/FreeSans.ttf"), 0));
+    }
+    v
+}
+
 pub fn apply(ctx: &egui::Context, dark: bool) {
     // Install both palettes before System can switch between the style slots.
     for theme in [egui::Theme::Light, egui::Theme::Dark] {
@@ -357,3 +511,71 @@ pub fn apply(ctx: &egui::Context, dark: bool) {
 pub fn color32(rgb: [u8; 3]) -> Color32 {
     Color32::from_rgb(rgb[0], rgb[1], rgb[2])
 }
+
+#[cfg(all(test, not(target_arch = "wasm32")))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fallback_faces_are_appended_to_every_family() {
+        // Structural: whatever loaded, the non-Latin fallback keys must sit at the end of each chain
+        // (after the Latin/default fonts) so Latin still wins for Latin text.
+        let fonts = font_definitions();
+        let fb_keys: Vec<String> = fallback_faces().into_iter().map(|(k, _)| k.to_string()).filter(|k| fonts.font_data.contains_key(k)).collect();
+        if fb_keys.is_empty() {
+            return; // no fallback face on this machine (minimal Linux)
+        }
+        for role in [UI, UI_BOLD, CELL, CELL_BOLD, CELL_ITALIC, CELL_BOLD_ITALIC, SERIF, MONO] {
+            let chain = fonts.families.get(&FontFamily::Name(role.into())).map(Vec::as_slice).unwrap_or_default();
+            assert!(chain.len() >= fb_keys.len(), "{role} chain too short for its fallback");
+            let tail = &chain[chain.len() - fb_keys.len()..];
+            assert_eq!(tail, fb_keys.as_slice(), "{role} must end its chain with the fallback faces");
+        }
+    }
+
+    #[test]
+    fn fallback_faces_load_each_file_once() {
+        // The region-merged Noto Sans CJK .ttc serves Han and Hangul; it must be loaded once, not
+        // once per script (tens of MB each).
+        let fonts = font_definitions();
+        let present: Vec<&Arc<FontData>> = fallback_faces().into_iter().filter_map(|(k, _)| fonts.font_data.get(k)).collect();
+        for (i, a) in present.iter().enumerate() {
+            for b in present.iter().skip(i + 1) {
+                assert!(a.font != b.font, "the same font file is loaded for two fallback scripts");
+            }
+        }
+    }
+
+    #[test]
+    fn fallback_covers_its_script() {
+        // Real render check, per script: the proportional family (what cells and the UI use) must
+        // resolve a sample string to a real glyph, not the tofu replacement char. Skips a script
+        // whose face is absent (a minimal machine) and fails if a face is present but not wired in.
+        let cases: [(&str, Vec<(String, u32)>, &str); 4] = [
+            ("sys-cjk-han", cjk_han(), "カテゴリ"),
+            ("sys-cjk-hangul", cjk_hangul(), "한글"),
+            ("sys-thai", thai(), "ทดสอบ"),
+            ("sys-hebrew", hebrew(), "עברית"),
+        ];
+        let ctx = egui::Context::default();
+        ctx.set_fonts(font_definitions());
+        let mut out = ctx.run_ui(egui::RawInput::default(), |_ui| {});
+        out.textures_delta.clear(); // nothing consumes the atlas in a headless test
+        // `has_glyphs` is a false negative when the face that owns the script is also the face that
+        // draws the replacement glyph (DejaVu Sans as the UI font owns Hebrew), so compare the laid
+        // out glyphs with the tofu glyph itself instead.
+        let font = FontId::proportional(14.0);
+        let glyphs = |text: &str| {
+            let galley = ctx.fonts_mut(|f| f.layout_no_wrap(text.to_owned(), font.clone(), Color32::BLACK));
+            galley.rows.iter().flat_map(|r| r.glyphs.iter().map(|g| g.uv_rect)).collect::<Vec<_>>()
+        };
+        let tofu = glyphs("\u{25FB}");
+        for (key, cands, sample) in cases {
+            if !cands.iter().any(|(p, _)| std::path::Path::new(p).exists()) {
+                continue; // no face for this script on this machine; nothing to assert
+            }
+            let drawn = glyphs(sample);
+            assert!(!drawn.is_empty() && drawn.iter().all(|g| !tofu.contains(g)), "{key} is present but {sample:?} still renders as tofu");
+        }
+    }
+}