mirror of
https://github.com/storytold/wordcraft.git
synced 2026-10-31 22:07:06 -06:00
4950690479
Indonesian spelling reads the installed Hunspell id_ID dictionary at runtime (Debian/Ubuntu hunspell-id, LibreOffice dict-id, the Dictionaries folder or WORDCRAFT_DICTIONARIES), like Czech, Polish and Portuguese: nothing is bundled, and without a dictionary Indonesian text gets no spelling marks. - ProofLang::Id for id, id-ID, ind and the legacy in tag - suggestions reuse the dictionary-driven Czech ranking (MAP/REP/TRY/KEY of the loaded dictionary), nothing Indonesian hard-coded - hyphenated reduplication accepted whole or by parts (kupu-kupu, anak-anak); hyphenated clitics after capitals/abbreviations (KTP-nya, se-Indonesia) - generic grammar only: repeated word (offers anak-anak first), space before punctuation, double space, with Indonesian messages - Review > Language, status bar, Set Proofing Language, File > Options > Proofing list Indonesian; name translated in every catalog - tests use a hand-written synthetic dictionary; engine end-to-end test Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Brandon Thomas <echelon@gmail.com>
880 lines
34 KiB
Rust
880 lines
34 KiB
Rust
//! Hyphenation, per proofing language ([`ProofLang`]).
|
||
//!
|
||
//! **English** (and languages without patterns of their own, as before languages were told
|
||
//! apart), lookup order for each word:
|
||
//! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list
|
||
//! by Grady Ward (~160k words), stored compactly in `assets/hyphenation/en-us.dic`; simple
|
||
//! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem;
|
||
//! 2. **Liang patterns** ([`Patterns::en_us`]) for words not in the dictionary — our own pattern
|
||
//! set, trained from the same list by DesignCraft's patgen (generated by DesignCraft's `hyphgen` example);
|
||
//! 3. a small syllable **heuristic** for words outside the pattern alphabet.
|
||
//!
|
||
//! **Polish**: the exception words, then the TeX Polish patterns ([`Patterns::pl`], by Hanna
|
||
//! Kołodziejska, Bogusław Jackowski and Marek Ryćko, MIT licence option), then a Polish syllable
|
||
//! heuristic for words with letters outside the Polish alphabet (`é`, `ü`…).
|
||
//!
|
||
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
|
||
//!
|
||
//! Then the paragraph's limits ([`Limits`]) are applied.
|
||
|
||
pub use crate::dict;
|
||
|
||
pub use crate::patterns;
|
||
|
||
use std::sync::OnceLock;
|
||
|
||
pub use crate::lang::ProofLang;
|
||
pub use dict::Dictionary;
|
||
pub use patterns::Patterns;
|
||
|
||
static EN_US_DIC: &[u8] = include_bytes!("../../../assets/hyphenation/en-us.dic");
|
||
static EN_US_PAT: &str = include_str!("../../../assets/hyphenation/en-us.pat");
|
||
/// hyph-utf8's `hyph-pl.tex`, unmodified (its licence header is in the file).
|
||
static PL_PAT: &str = include_str!("../../../assets/hyphenation/pl.pat");
|
||
/// hyph-utf8's `hyph-pt.tex`, unmodified (its BSD 3-clause header is in the file).
|
||
static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
|
||
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
|
||
/// notice is in the file's header).
|
||
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
|
||
|
||
impl Dictionary {
|
||
/// The bundled US English dictionary (decoded on first use).
|
||
pub fn en_us() -> &'static Dictionary {
|
||
static D: OnceLock<Dictionary> = OnceLock::new();
|
||
D.get_or_init(|| {
|
||
if EN_US_DIC.is_empty() {
|
||
return Dictionary::default();
|
||
}
|
||
Dictionary::from_bytes(EN_US_DIC).unwrap_or_else(|e| {
|
||
log::error!("{e}");
|
||
Dictionary::default()
|
||
})
|
||
})
|
||
}
|
||
}
|
||
|
||
impl Patterns {
|
||
/// The bundled US English patterns (parsed on first use).
|
||
pub fn en_us() -> &'static Patterns {
|
||
static P: OnceLock<Patterns> = OnceLock::new();
|
||
P.get_or_init(|| Patterns::parse(EN_US_PAT))
|
||
}
|
||
|
||
/// The bundled Polish patterns and exception words (parsed on first use).
|
||
pub fn pl() -> &'static Patterns {
|
||
static P: OnceLock<Patterns> = OnceLock::new();
|
||
P.get_or_init(|| Patterns::parse(PL_PAT))
|
||
}
|
||
|
||
/// The bundled Portuguese patterns and exception words (parsed on first use).
|
||
pub fn pt() -> &'static Patterns {
|
||
static P: OnceLock<Patterns> = OnceLock::new();
|
||
P.get_or_init(|| Patterns::parse(PT_PAT))
|
||
}
|
||
|
||
/// The bundled Italian patterns (parsed on first use; the file has no exception words).
|
||
pub fn it() -> &'static Patterns {
|
||
static P: OnceLock<Patterns> = OnceLock::new();
|
||
P.get_or_init(|| Patterns::parse(IT_PAT))
|
||
}
|
||
}
|
||
|
||
/// Hyphenation limits from the paragraph's Hyphenation settings.
|
||
#[derive(Clone, Copy, Debug)]
|
||
pub struct Limits {
|
||
pub min_word: usize,
|
||
pub after_first: usize,
|
||
pub before_last: usize,
|
||
pub capitalized: bool,
|
||
}
|
||
|
||
impl Default for Limits {
|
||
fn default() -> Self {
|
||
Limits { min_word: 5, after_first: 2, before_last: 2, capitalized: true }
|
||
}
|
||
}
|
||
|
||
/// Where a word's break points came from.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||
pub enum Source {
|
||
Dictionary,
|
||
Patterns,
|
||
Heuristic,
|
||
/// A language's syllable rules (Bulgarian, [`crate::bg`]).
|
||
Rules,
|
||
None,
|
||
}
|
||
|
||
fn is_apostrophe(c: char) -> bool {
|
||
matches!(c, '\'' | '’' | 'ʼ')
|
||
}
|
||
|
||
fn is_word_char(c: char) -> bool {
|
||
c.is_alphabetic() || is_apostrophe(c)
|
||
}
|
||
|
||
/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`).
|
||
fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||
let n = core.len();
|
||
if n < 2 {
|
||
return (vec![], Source::None);
|
||
}
|
||
if lang == ProofLang::Pl {
|
||
return core_breaks_pl(core);
|
||
}
|
||
// Bulgarian words follow the Bulgarian syllable rules; Latin-script words in a Bulgarian run
|
||
// (`WordCraft`, `email`) are hyphenated as English, as in any other language.
|
||
if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) {
|
||
return (crate::bg::points(core), Source::Rules);
|
||
}
|
||
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
|
||
return (crate::tr::hyphen::points(core), Source::Rules);
|
||
}
|
||
if lang == ProofLang::Pt {
|
||
return core_breaks_pt(core);
|
||
}
|
||
if lang == ProofLang::It {
|
||
return core_breaks_it(core);
|
||
}
|
||
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
|
||
return (crate::ru::hyphen::points(core), Source::Heuristic);
|
||
}
|
||
let dict = Dictionary::en_us();
|
||
let s: String = core.iter().collect();
|
||
let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::<Vec<_>>();
|
||
if let Some(m) = dict.get(&s) {
|
||
return (from_mask(m, n), Source::Dictionary);
|
||
}
|
||
// Inflections inherit the stem's breaks (the suffix adds no break of its own).
|
||
for suf in ["'s", "s", "es", "ed", "d"] {
|
||
if let Some(stem) = s.strip_suffix(suf)
|
||
&& stem.chars().count() >= 2
|
||
&& let Some(m) = dict.get(stem)
|
||
{
|
||
return (from_mask(m, stem.chars().count()), Source::Dictionary);
|
||
}
|
||
}
|
||
let pats = Patterns::en_us();
|
||
if !pats.is_empty()
|
||
&& let Some(p) = pats.points(core)
|
||
{
|
||
return (p, Source::Patterns);
|
||
}
|
||
(heuristic_points(core), Source::Heuristic)
|
||
}
|
||
|
||
/// Polish: exception words, then the patterns, then the Polish syllable heuristic.
|
||
fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
|
||
let pats = Patterns::pl();
|
||
let s: String = core.iter().collect();
|
||
if let Some(p) = pats.exception(&s) {
|
||
return (p.to_vec(), Source::Dictionary);
|
||
}
|
||
if !pats.is_empty()
|
||
&& let Some(p) = pats.points(core)
|
||
{
|
||
return (p, Source::Patterns);
|
||
}
|
||
(heuristic_points_pl(core), Source::Heuristic)
|
||
}
|
||
|
||
/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic.
|
||
fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
|
||
let pats = Patterns::pt();
|
||
let s: String = core.iter().collect();
|
||
if let Some(p) = pats.exception(&s) {
|
||
return (p.to_vec(), Source::Dictionary);
|
||
}
|
||
if !pats.is_empty()
|
||
&& let Some(p) = pats.points(core)
|
||
{
|
||
return (p, Source::Patterns);
|
||
}
|
||
(heuristic_points(core), Source::Heuristic)
|
||
}
|
||
|
||
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
|
||
/// then the generic heuristic.
|
||
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
|
||
let pats = Patterns::it();
|
||
let s: String = core.iter().collect();
|
||
if let Some(p) = pats.exception(&s) {
|
||
return (p.to_vec(), Source::Dictionary);
|
||
}
|
||
if !pats.is_empty()
|
||
&& let Some(p) = pats.points(core)
|
||
{
|
||
return (p, Source::Patterns);
|
||
}
|
||
(heuristic_points(core), Source::Heuristic)
|
||
}
|
||
|
||
/// Unrestricted break points of an English `word` ([`word_breaks_in`] with [`ProofLang::En`]).
|
||
pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
|
||
word_breaks_in(word, ProofLang::En)
|
||
}
|
||
|
||
/// Unrestricted break points of `word` in `lang` (char indices; a hyphen goes before that char),
|
||
/// handling case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter
|
||
/// run is hyphenated separately). No break is ever placed next to an apostrophe or non-letter.
|
||
pub fn word_breaks_in(word: &str, lang: ProofLang) -> (Vec<usize>, Source) {
|
||
let chars: Vec<char> = word.chars().collect();
|
||
let mut out = Vec::new();
|
||
let mut src = Source::None;
|
||
let mut i = 0;
|
||
while i < chars.len() {
|
||
if !is_word_char(chars[i]) {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
let mut j = i;
|
||
while j < chars.len() && is_word_char(chars[j]) {
|
||
j += 1;
|
||
}
|
||
let (mut a, mut b) = (i, j);
|
||
while a < b && is_apostrophe(chars[a]) {
|
||
a += 1;
|
||
}
|
||
while b > a && is_apostrophe(chars[b - 1]) {
|
||
b -= 1;
|
||
}
|
||
let core: Vec<char> = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect();
|
||
let (pts, s) = core_breaks(&core, lang);
|
||
if src == Source::None {
|
||
src = s;
|
||
}
|
||
out.extend(pts.into_iter().filter(|&p| p > 0 && p < core.len() && core[p - 1] != '\'' && core[p] != '\'').map(|p| a + p));
|
||
i = j;
|
||
}
|
||
(out, src)
|
||
}
|
||
|
||
/// Allowed hyphenation points in an English `word` ([`hyphen_points_in`] with [`ProofLang::En`]).
|
||
pub fn hyphen_points(word: &str, lim: &Limits) -> Vec<usize> {
|
||
hyphen_points_in(word, lim, ProofLang::En)
|
||
}
|
||
|
||
/// Allowed hyphenation points in `word`, a word in `lang`, as char indices (a hyphen goes before
|
||
/// that char), with the paragraph limits applied to each letter run.
|
||
pub fn hyphen_points_in(word: &str, lim: &Limits, lang: ProofLang) -> Vec<usize> {
|
||
let chars: Vec<char> = word.chars().collect();
|
||
let (pts, _) = word_breaks_in(word, lang);
|
||
if pts.is_empty() {
|
||
return pts;
|
||
}
|
||
let min_before = lim.after_first.max(1);
|
||
let min_after = lim.before_last.max(1);
|
||
// Counted once, so each point's checks take constant time however long the word: letters and
|
||
// uppercase letters before each char, and the letter run (trimmed of apostrophes) of each char.
|
||
let (mut letters, mut upper) = (Vec::with_capacity(chars.len() + 1), Vec::with_capacity(chars.len() + 1));
|
||
let (mut l, mut u) = (0usize, 0usize);
|
||
for c in &chars {
|
||
letters.push(l);
|
||
upper.push(u);
|
||
if c.is_alphabetic() {
|
||
l += 1;
|
||
u += usize::from(c.is_uppercase());
|
||
}
|
||
}
|
||
letters.push(l);
|
||
upper.push(u);
|
||
let mut run_of: Vec<(usize, usize)> = vec![(0, 0); chars.len()];
|
||
let mut i = 0;
|
||
while i < chars.len() {
|
||
let mut j = i;
|
||
while chars.get(j).is_some_and(|c| is_word_char(*c)) {
|
||
j += 1;
|
||
}
|
||
let (mut a, mut b) = (i, j);
|
||
while a < b && chars.get(a).is_some_and(|c| is_apostrophe(*c)) {
|
||
a += 1;
|
||
}
|
||
while b > a && chars.get(b - 1).is_some_and(|c| is_apostrophe(*c)) {
|
||
b -= 1;
|
||
}
|
||
for r in run_of.get_mut(i..j).into_iter().flatten() {
|
||
*r = (a, b);
|
||
}
|
||
i = j.max(i + 1);
|
||
}
|
||
let count = |v: &[usize], a: usize, b: usize| v.get(b).zip(v.get(a)).map_or(0, |(y, x)| y.saturating_sub(*x));
|
||
pts.into_iter()
|
||
.filter(|&p| {
|
||
// A point lies between two letters of one run; the char after it gives the run.
|
||
let Some(&(a, b)) = run_of.get(p) else { return false };
|
||
let n = count(&letters, a, b);
|
||
if n < lim.min_word.max(2) {
|
||
return false;
|
||
}
|
||
if n > 1 && count(&upper, a, b) == n {
|
||
return false;
|
||
}
|
||
if !lim.capitalized && chars.get(a).is_some_and(|c| c.is_uppercase()) {
|
||
return false;
|
||
}
|
||
count(&letters, a, p) >= min_before && count(&letters, p, b) >= min_after
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// `word` with `-` at every hyphenation point (tests and diagnostics).
|
||
pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
|
||
hyphenate_word_in(word, lim, ProofLang::En)
|
||
}
|
||
|
||
/// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics).
|
||
pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String {
|
||
let pts = hyphen_points_in(word, lim, lang);
|
||
let mut s = String::with_capacity(word.len() + pts.len());
|
||
for (i, ch) in word.chars().enumerate() {
|
||
if pts.contains(&i) {
|
||
s.push('-');
|
||
}
|
||
s.push(ch);
|
||
}
|
||
s
|
||
}
|
||
|
||
// ---------- heuristic fallback (words outside the pattern alphabet) ----------
|
||
|
||
/// Letters with a diacritic that are consonants (`ł`, `ż`, `ç`, `ñ`…); any other non-ASCII letter
|
||
/// counts as a vowel (`é`, `ø`, `ą`…), and so do letters of other scripts, which keeps the
|
||
/// heuristic from breaking them.
|
||
const NON_ASCII_CONSONANTS: &str = "łńśćźżçčďđğģķļľĺňñņřŕšşșťţțžßŋðþ";
|
||
|
||
fn is_vowel(c: char) -> bool {
|
||
let l = c.to_lowercase().next().unwrap_or(c);
|
||
if l.is_ascii() { matches!(l, 'a' | 'e' | 'i' | 'o' | 'u' | 'y') } else { l.is_alphabetic() && !NON_ASCII_CONSONANTS.contains(l) }
|
||
}
|
||
|
||
fn onset_pair(a: char, b: char) -> bool {
|
||
let (a, b) = (a.to_ascii_lowercase(), b.to_ascii_lowercase());
|
||
matches!(
|
||
(a, b),
|
||
('c' | 's' | 't' | 'p' | 'w' | 'g', 'h')
|
||
| ('b' | 'c' | 'd' | 'f' | 'g' | 'k' | 'p' | 't', 'r')
|
||
| ('b' | 'c' | 'f' | 'g' | 'k' | 'p' | 's', 'l')
|
||
| ('q', 'u')
|
||
| ('s', 't' | 'p' | 'c' | 'k')
|
||
)
|
||
}
|
||
|
||
/// Syllable heuristic: V-CV ("ty-po"), VC-CV ("hap-pen"), keeping onset clusters ("gra-phy").
|
||
fn heuristic_points(c: &[char]) -> Vec<usize> {
|
||
let n = c.len();
|
||
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
|
||
return vec![];
|
||
}
|
||
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
|
||
// Both parts need a sounded vowel: a trailing silent `e`, `ed` or `es` doesn't count. The part
|
||
// after `i` is `c[i..end]`, `end` before that ending; computed once, so each point is O(1).
|
||
let end = |i: usize| {
|
||
if n - i >= 2 && (c.ends_with(&['e', 'd']) || c.ends_with(&['e', 's'])) {
|
||
n - 2
|
||
} else if c.ends_with(&['e']) {
|
||
n - 1
|
||
} else {
|
||
n
|
||
}
|
||
};
|
||
// UTF-8 bytes before each char, and the first vowel at or after each char.
|
||
let mut bytes = Vec::with_capacity(n + 1);
|
||
let mut acc = 0usize;
|
||
for ch in c {
|
||
bytes.push(acc);
|
||
acc += ch.len_utf8();
|
||
}
|
||
bytes.push(acc);
|
||
let mut next_vowel = vec![n; n + 1];
|
||
for i in (0..n).rev() {
|
||
next_vowel[i] = if v[i] { i } else { next_vowel[i + 1] };
|
||
}
|
||
let mut out = vec![];
|
||
let mut vowel_before = false;
|
||
for i in 1..n {
|
||
vowel_before |= v[i - 1];
|
||
let ok = if v[i - 1] && !v[i] {
|
||
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
|
||
} else if !v[i - 1] && !v[i] {
|
||
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i] || (c[i - 1] == c[i] && i >= 2 && v[i - 2])
|
||
} else {
|
||
false
|
||
};
|
||
let e = end(i);
|
||
if ok && next_vowel[i] < e && bytes[e] - bytes[i] > 1 && vowel_before {
|
||
out.push(i);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Polish digraphs, one sound each: never split.
|
||
fn is_digraph_pl(a: char, b: char) -> bool {
|
||
matches!((a, b), ('c', 'h' | 'z') | ('d', 'z' | 'ź' | 'ż') | ('r' | 's', 'z'))
|
||
}
|
||
|
||
/// Polish syllable heuristic, for words with letters outside the Polish pattern alphabet (`é`,
|
||
/// `ü`…): one break between each pair of syllable nuclei — after a leading sonorant (`kon-ty`,
|
||
/// `Pol-ska`) or inside a doubled consonant (`Pöp-pel`), else before the consonant cluster
|
||
/// (`sio-stra`). Digraphs (`ch`, `cz`, `dz`, `dź`, `dż`, `rz`, `sz`) stay whole, and an `i` that
|
||
/// only softens the consonant before a vowel (`nia`, `sio`) stays with that consonant.
|
||
fn heuristic_points_pl(c: &[char]) -> Vec<usize> {
|
||
let n = c.len();
|
||
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
|
||
return vec![];
|
||
}
|
||
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
|
||
let at = |i: usize| v.get(i).copied().unwrap_or(false);
|
||
// A syllable nucleus: a vowel, except a softening `i` (after a consonant, before a vowel).
|
||
let nucleus: Vec<bool> = (0..n).map(|i| at(i) && !(c.get(i) == Some(&'i') && i > 0 && !at(i - 1) && at(i + 1))).collect();
|
||
let nuclei: Vec<usize> = (0..n).filter(|&i| nucleus.get(i).copied().unwrap_or(false)).collect();
|
||
let mut out = vec![];
|
||
for pair in nuclei.windows(2) {
|
||
let [a, b] = pair else { continue };
|
||
let (start, end) = (a + 1, *b);
|
||
if start >= end {
|
||
continue; // adjacent vowels (na-u-ka): no break inside a vowel pair here
|
||
}
|
||
// Consonant units of the cluster: digraphs and a softening `i` join their consonant.
|
||
let mut units: Vec<usize> = Vec::new();
|
||
let mut i = start;
|
||
while i < end {
|
||
units.push(i);
|
||
i += if i + 1 < end && c.get(i).zip(c.get(i + 1)).is_some_and(|(x, y)| is_digraph_pl(*x, *y)) { 2 } else { 1 };
|
||
if i < end && c.get(i) == Some(&'i') {
|
||
i += 1;
|
||
}
|
||
}
|
||
let sonorant = |i: usize| matches!(c.get(i), Some('l' | 'ł' | 'm' | 'n' | 'ń' | 'r' | 'j'));
|
||
let first_is_single = units.get(1).is_some_and(|&second| second == start + 1);
|
||
// A doubled consonant splits (`lek-ki`, `Pöp-pel`), as does a cluster after a sonorant.
|
||
let doubled = c.get(start).is_some_and(|x| c.get(start + 1) == Some(x));
|
||
out.push(if units.len() >= 2 && first_is_single && (sonorant(start) || doubled) { start + 1 } else { start });
|
||
}
|
||
out
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn h(w: &str) -> String {
|
||
hyphenate_word(w, &Limits::default())
|
||
}
|
||
|
||
#[test]
|
||
fn bundled_data_loads() {
|
||
assert!(Dictionary::en_us().len() > 150_000, "{}", Dictionary::en_us().len());
|
||
assert!(Patterns::en_us().len() > 1000, "{}", Patterns::en_us().len());
|
||
}
|
||
|
||
#[test]
|
||
fn dictionary_words() {
|
||
assert_eq!(h("happen"), "hap-pen");
|
||
assert_eq!(h("typography"), "ty-pog-ra-phy");
|
||
assert_eq!(h("composition"), "com-po-si-tion");
|
||
assert_eq!(h("hyphenation"), "hy-phen-a-tion");
|
||
assert_eq!(h("international"), "in-ter-na-tion-al");
|
||
assert_eq!(h("cat"), "cat");
|
||
assert!(!h("changed").contains('-'));
|
||
assert_eq!(h("NASA"), "NASA");
|
||
assert_eq!(word_breaks("publication").1, Source::Dictionary);
|
||
}
|
||
|
||
#[test]
|
||
fn case_apostrophes_and_punctuation() {
|
||
assert_eq!(h("Typography"), "Ty-pog-ra-phy");
|
||
assert_eq!(h("TYPOGRAPHY"), "TYPOGRAPHY");
|
||
assert_eq!(h("typography,"), "ty-pog-ra-phy,");
|
||
assert_eq!(h("“typography”"), "“ty-pog-ra-phy”");
|
||
assert_eq!(h("typography's"), "ty-pog-ra-phy's");
|
||
assert_eq!(h("typography’s"), "ty-pog-ra-phy’s");
|
||
assert_eq!(h("'typography'"), "'ty-pog-ra-phy'");
|
||
// Compounds: each part separately, never next to the hyphen.
|
||
assert_eq!(h("typography-composition"), "ty-pog-ra-phy-com-po-si-tion");
|
||
}
|
||
|
||
#[test]
|
||
fn inflections_inherit_the_stem() {
|
||
assert_eq!(h("ripples"), "rip-ples");
|
||
assert_eq!(h("gradients"), "gra-di-ents");
|
||
assert_eq!(h("restrained"), "re-strained");
|
||
}
|
||
|
||
#[test]
|
||
fn patterns_cover_unknown_words() {
|
||
// Not in the word list: the trained patterns decide.
|
||
let (pts, src) = word_breaks("hyperlocalization");
|
||
assert_eq!(src, Source::Patterns);
|
||
assert!(pts.len() >= 4, "{}", h("hyperlocalization"));
|
||
let (_, src) = word_breaks("zzyzx");
|
||
assert_ne!(src, Source::Dictionary);
|
||
}
|
||
|
||
#[test]
|
||
fn limits_are_respected() {
|
||
let lim = Limits { min_word: 5, after_first: 3, before_last: 3, capitalized: false };
|
||
for w in ["typography", "hyphenation", "justification", "Paragraph", "beautiful"] {
|
||
for p in hyphen_points(w, &lim) {
|
||
assert!(p >= 3 && p <= w.chars().count() - 3, "{w} {p}");
|
||
}
|
||
}
|
||
assert!(hyphen_points("Paragraph", &lim).is_empty());
|
||
let lim = Limits { min_word: 12, ..Limits::default() };
|
||
assert!(hyphen_points("typography", &lim).is_empty());
|
||
}
|
||
|
||
/// The CPU time `f` takes on this thread, in seconds. On Linux it is the scheduler's count of the
|
||
/// time the thread actually ran, so the time it waits while other work on a busy machine has the
|
||
/// CPU is left out; elsewhere, wall-clock time.
|
||
fn cpu_cost(f: &mut impl FnMut()) -> f64 {
|
||
let ran = || {
|
||
let s = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?;
|
||
s.split_whitespace().next()?.parse::<u64>().ok()
|
||
};
|
||
let (t, before) = (std::time::Instant::now(), ran());
|
||
f();
|
||
match (before, ran()) {
|
||
(Some(a), Some(b)) => b.saturating_sub(a) as f64 * 1e-9,
|
||
_ => t.elapsed().as_secs_f64(),
|
||
}
|
||
}
|
||
|
||
/// How much more `big` costs than `small`, for `big` an input `scale` times the size of `small`'s:
|
||
/// (small, big, big / small), ~`scale` when linear and ~`scale`² when quadratic. Each is the
|
||
/// cheapest of several runs after a warm-up, the runs alternating so a busy spell slows both sizes
|
||
/// alike; `small` is timed `scale` times over per run, so both timings are long against the clock's
|
||
/// resolution.
|
||
fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) {
|
||
small();
|
||
big();
|
||
let mut small_scaled = || (0..scale).for_each(|_| small());
|
||
let (mut t1, mut tk) = (f64::INFINITY, f64::INFINITY);
|
||
for _ in 0..5 {
|
||
t1 = t1.min(cpu_cost(&mut small_scaled) / scale as f64);
|
||
tk = tk.min(cpu_cost(&mut big));
|
||
}
|
||
(t1, tk, tk / t1.max(1e-9))
|
||
}
|
||
|
||
/// `hyphen_points`' limit checks as first written (a rescan of the letter run per point): the
|
||
/// reference for the linear version.
|
||
fn hyphen_points_ref(word: &str, lim: &Limits) -> Vec<usize> {
|
||
let chars: Vec<char> = word.chars().collect();
|
||
let (pts, _) = word_breaks(word);
|
||
let min_before = lim.after_first.max(1);
|
||
let min_after = lim.before_last.max(1);
|
||
let run_of = |p: usize| {
|
||
let mut a = p;
|
||
while a > 0 && is_word_char(chars[a - 1]) {
|
||
a -= 1;
|
||
}
|
||
let mut b = p;
|
||
while b < chars.len() && is_word_char(chars[b]) {
|
||
b += 1;
|
||
}
|
||
while a < b && is_apostrophe(chars[a]) {
|
||
a += 1;
|
||
}
|
||
while b > a && is_apostrophe(chars[b - 1]) {
|
||
b -= 1;
|
||
}
|
||
(a, b)
|
||
};
|
||
pts.into_iter()
|
||
.filter(|&p| {
|
||
let (a, b) = run_of(p);
|
||
let run = &chars[a..b];
|
||
let letters = run.iter().filter(|c| c.is_alphabetic()).count();
|
||
if letters < lim.min_word.max(2) {
|
||
return false;
|
||
}
|
||
if letters > 1 && run.iter().filter(|c| c.is_alphabetic()).all(|c| c.is_uppercase()) {
|
||
return false;
|
||
}
|
||
if !lim.capitalized && run[0].is_uppercase() {
|
||
return false;
|
||
}
|
||
let before = chars[a..p].iter().filter(|c| c.is_alphabetic()).count();
|
||
let after = chars[p..b].iter().filter(|c| c.is_alphabetic()).count();
|
||
before >= min_before && after >= min_after
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// `heuristic_points` as first written (the right part rebuilt per point): the reference.
|
||
fn heuristic_ref(c: &[char]) -> Vec<usize> {
|
||
let n = c.len();
|
||
if n < 4 || !c.iter().all(|c| c.is_alphabetic()) {
|
||
return vec![];
|
||
}
|
||
let v: Vec<bool> = c.iter().map(|&x| is_vowel(x)).collect();
|
||
let mut out = vec![];
|
||
for i in 1..n {
|
||
let ok = if v[i - 1] && !v[i] {
|
||
(i + 1 < n && v[i + 1]) || (i + 2 < n && onset_pair(c[i], c[i + 1]) && v[i + 2])
|
||
} else if !v[i - 1] && !v[i] {
|
||
i >= 2 && v[i - 2] && i + 1 < n && v[i + 1] && !onset_pair(c[i - 1], c[i]) && c[i - 1] != c[i]
|
||
|| (c[i - 1] == c[i] && i >= 2 && v[i - 2])
|
||
} else {
|
||
false
|
||
};
|
||
let right: String = c[i..].iter().collect();
|
||
let core = right.strip_suffix("ed").or_else(|| right.strip_suffix("es")).or_else(|| right.strip_suffix('e')).unwrap_or(&right);
|
||
if ok && core.chars().any(is_vowel) && core.len() > 1 && v[..i].iter().any(|x| *x) {
|
||
out.push(i);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
#[test]
|
||
fn linear_limits_and_heuristic_match_the_reference() {
|
||
let long = "hyphenation".repeat(40);
|
||
let words = [
|
||
"",
|
||
"x",
|
||
"''",
|
||
"a'b'c",
|
||
"cat",
|
||
"happen",
|
||
"hyphenation",
|
||
"Typography's",
|
||
"TYPOGRAPHY",
|
||
"Typography",
|
||
"typography’s",
|
||
"'typography'",
|
||
"“typography”,",
|
||
"typography-composition",
|
||
"don't",
|
||
"rock'n'roll",
|
||
"naïveté",
|
||
"crème-brûlée",
|
||
"Ünïcödé",
|
||
"ÉTUDE",
|
||
"strengths",
|
||
"ee",
|
||
"bed",
|
||
"hopped",
|
||
"rhythmed",
|
||
"aeiouy",
|
||
"xxyyzz",
|
||
"bookkeeper",
|
||
"ʼtwas",
|
||
"mid-1990s",
|
||
"twenty-one",
|
||
"école",
|
||
"Ålesund",
|
||
"smørrebrød",
|
||
"hopes",
|
||
"hoped",
|
||
"taed",
|
||
"teeed",
|
||
&long,
|
||
];
|
||
for w in words {
|
||
for min_word in [2, 5, 7] {
|
||
for (after_first, before_last) in [(1, 1), (2, 2), (3, 2), (2, 4)] {
|
||
for capitalized in [true, false] {
|
||
let lim = Limits { min_word, after_first, before_last, capitalized };
|
||
assert_eq!(hyphen_points(w, &lim), hyphen_points_ref(w, &lim), "{w:?} {lim:?}");
|
||
}
|
||
}
|
||
}
|
||
let c: Vec<char> = w.to_lowercase().chars().collect();
|
||
assert_eq!(heuristic_points(&c), heuristic_ref(&c), "{w:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn hyphenating_a_long_word_is_linear() {
|
||
// A word of n letters against one of 8n: linear ~8x the cost, the old rescans ~64x. One in
|
||
// the pattern alphabet, one outside it (the syllable heuristic).
|
||
let pattern = |k: usize| "hyphenation".repeat(k);
|
||
let heuristic = |k: usize| "smørrebrød".repeat(k);
|
||
assert_eq!(word_breaks(&heuristic(2)).1, Source::Heuristic);
|
||
// Each timing repeats the call so the shorter one is milliseconds, well above timer noise.
|
||
let ratios: Vec<(String, f64)> = [(pattern as fn(usize) -> String, 20), (heuristic, 40)]
|
||
.into_iter()
|
||
.map(|(word, repeat)| {
|
||
let (small, big) = (word(400), word(8 * 400));
|
||
let time = |w: &str| (0..repeat).for_each(|_| drop(hyphen_points(w, &Limits::default())));
|
||
let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big));
|
||
let what = format!("{}…: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}", word(1).chars().take(10).collect::<String>());
|
||
eprintln!("{what}");
|
||
(what, ratio)
|
||
})
|
||
.collect();
|
||
for (what, ratio) in ratios {
|
||
assert!(ratio < 24.0, "8x the letters, not linear: {what}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn heuristic_fallback() {
|
||
let c: Vec<char> = "happen".chars().collect();
|
||
assert_eq!(heuristic_points(&c), vec![3]);
|
||
}
|
||
|
||
#[test]
|
||
fn heuristic_knows_consonants_with_diacritics() {
|
||
// `ł`, `ż`, `ç` are consonants (every non-ASCII letter used to count as a vowel).
|
||
assert!(!is_vowel('ł') && !is_vowel('Ż') && !is_vowel('ç') && is_vowel('é') && is_vowel('ą') && is_vowel('ø'));
|
||
let c: Vec<char> = "façade".chars().collect();
|
||
assert_eq!(heuristic_points(&c), vec![2]);
|
||
}
|
||
|
||
fn pl(w: &str) -> String {
|
||
hyphenate_word_in(w, &Limits::default(), ProofLang::Pl)
|
||
}
|
||
|
||
#[test]
|
||
fn polish_patterns_load() {
|
||
let p = Patterns::pl();
|
||
assert!(p.len() > 4000, "{}", p.len());
|
||
assert_eq!(p.exception_count(), 20);
|
||
assert_eq!(p.exception("nigdy"), Some(&[2][..]));
|
||
}
|
||
|
||
fn pt(w: &str) -> String {
|
||
hyphenate_word_in(w, &Limits::default(), ProofLang::Pt)
|
||
}
|
||
|
||
fn it(w: &str) -> String {
|
||
hyphenate_word_in(w, &Limits::default(), ProofLang::It)
|
||
}
|
||
|
||
#[test]
|
||
fn italian_patterns_load() {
|
||
let p = Patterns::it();
|
||
assert!(p.len() > 300, "{}", p.len());
|
||
// The file has no exception words.
|
||
assert_eq!(p.exception_count(), 0);
|
||
}
|
||
|
||
#[test]
|
||
fn italian_words_hyphenate_by_italian_rules() {
|
||
for (w, want) in [
|
||
("famiglia", "fa-mi-glia"),
|
||
("finestra", "fi-ne-stra"),
|
||
("bicicletta", "bi-ci-clet-ta"),
|
||
("passeggiata", "pas-seg-gia-ta"),
|
||
("caffè", "caf-fè"),
|
||
("tavolo", "ta-vo-lo"),
|
||
("bambino", "bam-bi-no"),
|
||
("scuola", "scuo-la"),
|
||
("notte", "not-te"),
|
||
("giorno", "gior-no"),
|
||
("acqua", "ac-qua"),
|
||
// Under five letters (`Limits::min_word`): no break.
|
||
("casa", "casa"),
|
||
("pane", "pane"),
|
||
("libro", "li-bro"),
|
||
("re", "re"),
|
||
("più", "più"),
|
||
] {
|
||
assert_eq!(it(w), want, "{w}");
|
||
}
|
||
assert_eq!(word_breaks_in("famiglia", ProofLang::It).1, Source::Patterns);
|
||
}
|
||
|
||
#[test]
|
||
fn portuguese_patterns_load() {
|
||
let p = Patterns::pt();
|
||
assert!(p.len() > 300, "{}", p.len());
|
||
// `hard-ware`, `soft-ware` — and the file's last pattern line, a bare `1-`, which the
|
||
// reader counts as a (never-matching) exception because it contains a hyphen.
|
||
assert_eq!(p.exception_count(), 3);
|
||
assert_eq!(p.exception("hardware"), Some(&[4][..]));
|
||
assert_eq!(p.exception("software"), Some(&[4][..]));
|
||
}
|
||
|
||
#[test]
|
||
fn portuguese_words_hyphenate_by_portuguese_rules() {
|
||
for (w, want) in [
|
||
("problema", "pro-ble-ma"),
|
||
("coração", "co-ra-ção"),
|
||
("paralelepípedo", "pa-ra-le-le-pí-pe-do"),
|
||
("computador", "com-pu-ta-dor"),
|
||
("escrever", "es-cre-ver"),
|
||
("saudade", "sau-da-de"),
|
||
("extraordinário", "ex-tra-or-di-ná-rio"),
|
||
("automático", "au-to-má-ti-co"),
|
||
// One syllable: no break.
|
||
("flor", "flor"),
|
||
("bem", "bem"),
|
||
] {
|
||
assert_eq!(pt(w), want, "{w}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn polish_words_hyphenate_by_polish_rules() {
|
||
for (w, want) in [
|
||
("konstytucja", "kon-sty-tu-cja"),
|
||
("konstytucyjnego", "kon-sty-tu-cyj-ne-go"),
|
||
("Rzeczpospolita", "Rzecz-po-spo-li-ta"),
|
||
("przedsiębiorstwo", "przed-się-bior-stwo"),
|
||
("pięćdziesięciu", "pięć-dzie-się-ciu"),
|
||
("dziewczyna", "dziew-czy-na"),
|
||
("książka", "książ-ka"),
|
||
("Warszawa", "War-sza-wa"),
|
||
("szczęście", "szczę-ście"),
|
||
("rzeczywistość", "rze-czy-wi-stość"),
|
||
("najpiękniejszy", "naj-pięk-niej-szy"),
|
||
("wyobraźnia", "wy-obraź-nia"),
|
||
("gżegżółka", "gżeg-żół-ka"),
|
||
// One syllable, or a single letter left over: no break.
|
||
("chrząszcz", "chrząszcz"),
|
||
("źdźbło", "źdźbło"),
|
||
("obiad", "obiad"),
|
||
// The pattern file's exception words.
|
||
("nigdy", "ni-gdy"),
|
||
("przynajmniej", "przy-naj-mniej"),
|
||
("skądinąd", "skąd-inąd"),
|
||
// Compounds: each part, never next to the hyphen.
|
||
("biało-czerwony", "bia-ło-czer-wo-ny"),
|
||
] {
|
||
assert_eq!(pl(w), want, "{w}");
|
||
}
|
||
assert_eq!(word_breaks_in("nigdy", ProofLang::Pl).1, Source::Dictionary);
|
||
assert_eq!(word_breaks_in("konstytucja", ProofLang::Pl).1, Source::Patterns);
|
||
}
|
||
|
||
#[test]
|
||
fn polish_heuristic_for_foreign_letters() {
|
||
// Letters outside the Polish alphabet: the Polish syllable heuristic, digraphs kept whole.
|
||
assert_eq!(word_breaks_in("Pöppelmann", ProofLang::Pl).1, Source::Heuristic);
|
||
assert_eq!(pl("Pöppelmann"), "Pöp-pel-mann");
|
||
assert_eq!(pl("Müllerowi"), "Mül-le-ro-wi");
|
||
let c: Vec<char> = "kawiarnié".chars().collect();
|
||
assert_eq!(heuristic_points_pl(&c), vec![2, 6], "ka-wiar-nié: `wi` and `ni` soften, `rn` splits after the sonorant");
|
||
let c: Vec<char> = "szczérze".chars().collect();
|
||
assert_eq!(heuristic_points_pl(&c), vec![5], "szczé-rze: `rz` stays whole");
|
||
}
|
||
|
||
#[test]
|
||
fn languages_without_patterns_keep_the_english_ones() {
|
||
for w in ["hyphenation", "typography", "internationalization"] {
|
||
for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id] {
|
||
assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}");
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn bulgarian_words_follow_the_bulgarian_rules() {
|
||
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
|
||
assert_eq!(bg("сричкопренасяне"), "срич-ко-пре-на-ся-не");
|
||
assert_eq!(bg("Таблица"), "Та-бли-ца");
|
||
// A single letter is never left on a line (the paragraph limits): България, not Бъл-га-ри-я.
|
||
assert_eq!(bg("България"), "Бъл-га-рия");
|
||
assert_eq!(bg("„шофьор“,"), "„шо-фьор“,");
|
||
assert_eq!(word_breaks_in("таблица", ProofLang::Bg).1, Source::Rules);
|
||
// Latin words in a Bulgarian run are hyphenated as English.
|
||
for w in ["publication", "WordCraft"] {
|
||
assert_eq!(hyphen_points_in(w, &Limits::default(), ProofLang::Bg), hyphen_points(w, &Limits::default()), "{w}");
|
||
}
|
||
}
|
||
}
|