From e161878e4c7744fa06f9a2eede8103cb4c296355 Mon Sep 17 00:00:00 2001 From: ROSSA AI <132106533+rossaai@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:21:25 -0500 Subject: [PATCH] Hyphenation: Spanish text is hyphenated with Javier Bezos's patterns (#222) Text set in Spanish was not hyphenated at all. It now uses Javier Bezos's Liang patterns, hyph-es.tex 5.0 from hyph-utf8 under MIT, through the hypher crate 0.1.8 with only its alloc and spanish features. The full feature, which embeds 48 languages under several licences, stays off. - hyphen::Lang names the hyphenator for a run: English, or a hypher language. Lang::for_language maps a language name or locale code. - word_breaks_in and hyphen_points_in take the language. word_breaks and hyphen_points stay as the English entry points. - Composition hyphenates a non-English run whose language has patterns and skips the rest. The word cache key includes the language. - NOTICE carries the Bezos copyright line and the MIT text. Co-authored-by: Claude Opus 5.5 Co-authored-by: Brandon Thomas --- Cargo.lock | 7 +++ NOTICE | 14 ++++++ ROADMAP.md | 2 +- crates/compose/Cargo.toml | 1 + crates/compose/src/hyphen/mod.rs | 80 ++++++++++++++++++++++++++++++-- crates/compose/src/lib.rs | 32 +++++++++---- crates/compose/src/tests.rs | 21 +++++++++ 7 files changed, 141 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c683910c2..3fc1e3274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -911,6 +911,7 @@ dependencies = [ "designcraft-doc", "designcraft-fonts", "designcraft-geom", + "hypher", "log", "miniz_oxide 0.8.9", "proptest", @@ -2085,6 +2086,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hypher" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f2b4f6b5da34d400d4c112eadcc14512cc5d04307a8eee182586d9bebd7ce6" + [[package]] name = "icu_collections" version = "2.3.0" diff --git a/NOTICE b/NOTICE index b1d382b04..8b67a99e1 100644 --- a/NOTICE +++ b/NOTICE @@ -18,6 +18,20 @@ list (author, source, license) is in ASSETS.md. - English hyphenation word list (assets/hyphenation/en-us.dic) — derived from the Moby Hyphenator II list by Grady Ward, public domain (Project Gutenberg eBook #3204). The hyphenation patterns (assets/hyphenation/en-us.pat) are trained from it by DesignCraft and released as CC0-1.0. +- Spanish hyphenation patterns, embedded through the hypher crate (feature "spanish"): + hyph-es.tex 5.0 from hyph-utf8, https://github.com/hyphenation/tex-hyphen. + Copyright (C) 1993, 1997 Javier Bezos, 2001-2019 Javier Bezos, CervanTeX. MIT License: + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: The above copyright notice + and this permission notice shall be included in all copies or substantial portions of the + Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE + AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - Japanese fonts (craft-fonts): builds made with craft-fonts (CRAFT_FONTS_DIR; all official releases) embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md (BIZ UDPGothic, BIZ UDMincho, diff --git a/ROADMAP.md b/ROADMAP.md index bb517b5a3..eaa813d99 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -91,7 +91,7 @@ DesignCraft aims at full Adobe InDesign parity — and to be better: faster, ope - Edit ▸ Menus (`window.hideMenuItem`): hide menu items; menus with hidden items end in Show All Menu Items. - View ▸ Separations Preview (`view.separations`): one process plate as ink density, or areas over an ink limit in red (a GCR preview of the rendered view; spot plates and the flattener preview aren't there). - Separations Preview plates now come from each object's own colour (CMYK as authored; RGB and images through the colour settings, images via a cached 17³ table), so pure black shows only on the black plate. -- Language (Character panel ▸ Language, Properties ▸ Character, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, Japanese and Traditional Chinese 「…」, Simplified Chinese “…”), the font's localized forms (OpenType `locl`: Turkish i, Bulgarian Cyrillic, Japanese / Chinese / Korean glyph variants) follow it too, and so does the fallback for CJK characters the font lacks (with Draw Missing Glyphs from Fallback Fonts on): Japanese, Simplified Chinese, Traditional Chinese and Korean each try their own installed system fonts (Japanese starts with the craft-fonts Japanese faces, Mincho first, when built with them; Hangul, kana and Bopomofo pick their language's fonts whatever the language). Language names from IDML keep their spelling; locale codes as InDesign writes them for some languages (`de_DE_2006`, `nl_NL_2005`; any `ll`, `ll_CC`, `ll-CC`) are their language and region for all of the above, and Chinese is also recognised in its common spellings ("Simplified Chinese", "Chinese (Traditional)", `zh_CN`, `zh-Hant`). Hyphenation dictionaries for other languages aren't bundled yet. +- Language (Character panel ▸ Language, Properties ▸ Character, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, Japanese and Traditional Chinese 「…」, Simplified Chinese “…”), the font's localized forms (OpenType `locl`: Turkish i, Bulgarian Cyrillic, Japanese / Chinese / Korean glyph variants) follow it too, and so does the fallback for CJK characters the font lacks (with Draw Missing Glyphs from Fallback Fonts on): Japanese, Simplified Chinese, Traditional Chinese and Korean each try their own installed system fonts (Japanese starts with the craft-fonts Japanese faces, Mincho first, when built with them; Hangul, kana and Bopomofo pick their language's fonts whatever the language). Language names from IDML keep their spelling; locale codes as InDesign writes them for some languages (`de_DE_2006`, `nl_NL_2005`; any `ll`, `ll_CC`, `ll-CC`) are their language and region for all of the above, and Chinese is also recognised in its common spellings ("Simplified Chinese", "Chinese (Traditional)", `zh_CN`, `zh-Hant`). Spanish is hyphenated with Javier Bezos's Liang patterns (`hyph-es.tex` 5.0, MIT) through the `hypher` crate; hyphenation for other languages isn't bundled yet. - Books (`book.new/open/add/remove/list/styleSource/paginate/syncStyles/exportPdf`, Book panel): `.dcbook` files, continuous page numbering, style sync from a style source, one merged PDF. - Track Changes (`changes.track/list/acceptAll/rejectAll`, Type ▸ Track Changes): typing is marked added (highlighted on screen), deletions are kept but hidden from layout and exports until accepted or rejected. Per-change accept/reject and authors aren't there yet. - Track Changes: per-change Accept / Reject (`changes.accept`, `changes.reject`) and a Track Changes panel listing every change. diff --git a/crates/compose/Cargo.toml b/crates/compose/Cargo.toml index 83ba81835..012868835 100644 --- a/crates/compose/Cargo.toml +++ b/crates/compose/Cargo.toml @@ -18,6 +18,7 @@ designcraft-fonts = { workspace = true } log = { workspace = true } regex = { workspace = true } miniz_oxide = "0.8" +hypher = { version = "0.1.8", default-features = false, features = ["alloc", "spanish"] } [dev-dependencies] designcraft-fonts = { workspace = true, features = ["testing"] } diff --git a/crates/compose/src/hyphen/mod.rs b/crates/compose/src/hyphen/mod.rs index 23d182aeb..22644eb70 100644 --- a/crates/compose/src/hyphen/mod.rs +++ b/crates/compose/src/hyphen/mod.rs @@ -1,6 +1,8 @@ -//! English hyphenation. +//! Hyphenation. English uses the bundled dictionary and patterns below. Other languages use the +//! Liang patterns compiled into `hypher`, one cargo feature per language: Spanish is Javier Bezos's +//! `hyph-es.tex` 5.0 from hyph-utf8, MIT (see NOTICE). //! -//! Lookup order for each word: +//! English lookup order for each word: //! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list //! by Grady Ward (~160k words), stored compactly in `assets/hyphenation/en-us.dic`; simple //! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem; @@ -61,6 +63,33 @@ impl Default for Limits { } } +/// The hyphenator for a run of text. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Lang { + English, + Patterns(hypher::Lang), +} + +impl Lang { + /// The hyphenator for a language name or locale code (see [`designcraft_doc::language_tag`]). + /// None when the language has no bundled hyphenation. + pub fn for_language(language: &str) -> Option { + if language.starts_with("English") { + return Some(Lang::English); + } + let tag = designcraft_doc::language_tag(language)?; + let sub = designcraft_doc::language_subtag(tag); + if sub == "en" { + return Some(Lang::English); + } + let b = sub.as_bytes(); + if b.len() != 2 { + return None; + } + hypher::Lang::from_iso([b[0], b[1]]).map(Lang::Patterns) + } +} + /// Where a word's break points came from. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Source { @@ -79,11 +108,25 @@ fn is_word_char(c: char) -> bool { } /// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`). -fn core_breaks(core: &[char]) -> (Vec, Source) { +fn core_breaks(core: &[char], lang: Lang) -> (Vec, Source) { let n = core.len(); if n < 2 { return (vec![], Source::None); } + if let Lang::Patterns(l) = lang { + if core.contains(&'\'') { + return (vec![], Source::None); + } + let s: String = core.iter().collect(); + let mut out = Vec::new(); + let mut at = 0; + let syl: Vec<&str> = hypher::hyphenate_bounded(&s, l, 1, 1).collect(); + for part in &syl[..syl.len().saturating_sub(1)] { + at += part.chars().count(); + out.push(at); + } + return (out, Source::Patterns); + } let dict = Dictionary::en_us(); let s: String = core.iter().collect(); let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::>(); @@ -112,6 +155,11 @@ fn core_breaks(core: &[char]) -> (Vec, Source) { /// case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter run is /// hyphenated separately). No break is ever placed next to an apostrophe or non-letter. pub fn word_breaks(word: &str) -> (Vec, Source) { + word_breaks_in(word, Lang::English) +} + +/// [`word_breaks`] for a language. +pub fn word_breaks_in(word: &str, lang: Lang) -> (Vec, Source) { let chars: Vec = word.chars().collect(); let mut out = Vec::new(); let mut src = Source::None; @@ -133,7 +181,7 @@ pub fn word_breaks(word: &str) -> (Vec, Source) { b -= 1; } let core: Vec = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect(); - let (pts, s) = core_breaks(&core); + let (pts, s) = core_breaks(&core, lang); if src == Source::None { src = s; } @@ -146,8 +194,13 @@ pub fn word_breaks(word: &str) -> (Vec, Source) { /// Allowed hyphenation points in `word` as char indices (a hyphen goes before that char), with /// the paragraph limits applied to each letter run. pub fn hyphen_points(word: &str, lim: &Limits) -> Vec { + hyphen_points_in(word, lim, Lang::English) +} + +/// [`hyphen_points`] for a language. +pub fn hyphen_points_in(word: &str, lim: &Limits, lang: Lang) -> Vec { let chars: Vec = word.chars().collect(); - let (pts, _) = word_breaks(word); + let (pts, _) = word_breaks_in(word, lang); if pts.is_empty() { return pts; } @@ -319,6 +372,23 @@ mod tests { assert!(hyphen_points("typography", &lim).is_empty()); } + #[test] + fn spanish_uses_the_bezos_patterns() { + let es = Lang::for_language("Spanish").unwrap(); + assert_eq!(es, Lang::for_language("es_ES").unwrap()); + let h = |w: &str| { + let pts = hyphen_points_in(w, &Limits::default(), es); + w.chars().enumerate().map(|(i, c)| if pts.contains(&i) { format!("-{c}") } else { c.to_string() }).collect::() + }; + assert_eq!(h("transición"), "tran-si-ción"); + assert_eq!(h("pingüino"), "pin-güino"); + assert_eq!(h("Construcción"), "Cons-truc-ción"); + assert_eq!(h("desesperación"), "de-ses-pe-ra-ción"); + assert_eq!(h("teatro"), "tea-tro", "a vowel run stays whole"); + assert_eq!(h("calle"), "ca-lle", "ll stays whole"); + assert!(Lang::for_language("French").is_none(), "only languages built in"); + } + #[test] fn heuristic_fallback() { let c: Vec = "happen".chars().collect(); diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index 8f34d1177..b48531d93 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -2284,9 +2284,14 @@ pub fn is_english(language: &str) -> bool { language.starts_with("English") || designcraft_doc::language_tag(language).is_some_and(|t| designcraft_doc::language_subtag(t) == "en") } -/// Byte ranges of paragraph `prange` set in a language other than English (no English -/// hyphenation there). -fn foreign_ranges(doc: &Document, story: &designcraft_doc::Story, prange: Range, base: &designcraft_doc::CharProps) -> Vec> { +/// Byte ranges of paragraph `prange` set in a language other than English, with the hyphenator +/// for that language (None: not hyphenated). +fn foreign_ranges( + doc: &Document, + story: &designcraft_doc::Story, + prange: Range, + base: &designcraft_doc::CharProps, +) -> Vec<(Range, Option)> { if is_english(&base.language) && story.runs().all(|(_, f)| f.over.language.as_deref().is_none_or(is_english) && f.style == designcraft_doc::story::NO_CHAR_STYLE) { @@ -2295,12 +2300,19 @@ fn foreign_ranges(doc: &Document, story: &designcraft_doc::Story, prange: Range< story .runs() .filter(|(r, _)| r.start < prange.end && r.end > prange.start) - .filter(|(_, f)| !is_english(&doc.styles.resolve_char(base, f).language)) - .map(|(r, _)| r) + .map(|(r, f)| (r, doc.styles.resolve_char(base, f).language.clone())) + .filter(|(_, l)| !is_english(l)) + .map(|(r, l)| (r, hyphen::Lang::for_language(&l))) .collect() } -fn hyphenation_points(text: &str, glyphs: &[Glyph], pp: &ParaProps, exceptions: &HashMap>, foreign: &[Range]) -> Vec { +fn hyphenation_points( + text: &str, + glyphs: &[Glyph], + pp: &ParaProps, + exceptions: &HashMap>, + foreign: &[(Range, Option)], +) -> Vec { let mut out = vec![false; glyphs.len()]; if !pp.hyphenate { return out; @@ -2312,7 +2324,7 @@ fn hyphenation_points(text: &str, glyphs: &[Glyph], pp: &ParaProps, exceptions: capitalized: pp.hyph_capitalized, }; // Words repeat a lot: remember their points per set of limits. - static CACHE: Mutex, Box<[usize]>>)>> = Mutex::new(Vec::new()); + static CACHE: Mutex), Box<[usize]>>)>> = Mutex::new(Vec::new()); let key = (lim.min_word, lim.after_first, lim.before_last, lim.capitalized); let mut guard = CACHE.lock().unwrap_or_else(|e| e.into_inner()); let idx = match guard.iter().position(|e| e.0 == key) { @@ -2351,13 +2363,13 @@ fn hyphenation_points(text: &str, glyphs: &[Glyph], pp: &ParaProps, exceptions: let word = &text[a..b]; // Do not hyphenate the paragraph's last word unless allowed. let is_last_word = !pp.hyph_last_word && glyphs[j..].iter().all(|g| !g.is_letter()); - let in_foreign = foreign.iter().any(|r| r.contains(&a)); - if !is_last_word && !in_foreign { + let lang = foreign.iter().find(|(r, _)| r.contains(&a)).map_or(Some(hyphen::Lang::English), |(_, l)| *l); + if !is_last_word && let Some(lang) = lang { let user = (!exceptions.is_empty()).then(|| exceptions.get(&word.to_lowercase())).flatten(); let pts: &[usize] = match user { // User dictionary exceptions win over the patterns and the built-in list. Some(v) => v, - None => cache.entry(word.into()).or_insert_with(|| hyphen::hyphen_points(word, &lim).into_boxed_slice()), + None => cache.entry((lang, word.into())).or_insert_with(|| hyphen::hyphen_points_in(word, &lim, lang).into_boxed_slice()), }; for &p in pts.iter() { // char index p → byte → glyph whose cluster ends at that byte. diff --git a/crates/compose/src/tests.rs b/crates/compose/src/tests.rs index 849d3a4e4..70b0198ff 100644 --- a/crates/compose/src/tests.rs +++ b/crates/compose/src/tests.rs @@ -1107,6 +1107,27 @@ fn only_english_text_gets_english_hyphenation() { assert_eq!(hyphenated(&d2, sid), 0, "French isn't hyphenated with English rules"); } +#[test] +fn spanish_text_gets_spanish_hyphenation() { + let text = "La transición democrática fue una construcción colectiva extraordinariamente compleja y desesperadamente necesaria."; + let narrow = Rect::new(36.0, 36.0, 120.0, 700.0); + let (mut d, sid, _) = doc_with(text, narrow, ParaAttrs::default()); + let st = d.story_mut(sid).unwrap(); + let n = st.len(); + st.format_chars(0..n, |f| f.over.language = Some("Spanish".into())); + let cs = compose_story(&d, sid, &ComposeOptions::default()); + let lines = all_lines(&cs); + let hy: Vec<&str> = lines.iter().filter(|l| l.hyphenated).map(|l| text[l.range.clone()].trim_end()).collect(); + assert!(!hy.is_empty(), "Spanish hyphenates"); + for l in &hy { + let last: String = l.chars().rev().take_while(|c| c.is_alphabetic()).collect::>().into_iter().rev().collect(); + let whole = text[text.find(l).unwrap() + l.len() - last.len()..].split(|c: char| !c.is_alphabetic()).next().unwrap(); + let es = hyphen::Lang::for_language("Spanish").unwrap(); + let pts = hyphen::hyphen_points_in(whole, &hyphen::Limits::default(), es); + assert!(pts.contains(&last.chars().count()), "break {last}- in {whole} is a Bezos point"); + } +} + #[test] fn cjk_aki_adds_space_without_scaling_outlines() { let (mut d, sid, _) = doc_with("AB", Rect::new(0.0, 0.0, 300.0, 100.0), ParaAttrs::default());