From 695b2688d8c88aacf1e491c99044cfae2eb8ed05 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:19:25 -0100 Subject: [PATCH 01/21] Proofing options reach the Russian and Turkish checks too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3966 made Ignore words in UPPERCASE and Ignore words that contain numbers reach six languages' own checks, one copy at a time, and Russian and Turkish were left out. lib.rs still called `ru::is_correct(word)` and `tr::is_correct(word)`, which accept words with digits (both) and short words in capitals (Russian) whatever the options say, so turning either option off changed nothing in those languages. Both now have `is_correct_with_options`, like the others. Russian's rule for a part in capitals (`ИИ-консультант`) follows the option too, as Indonesian's does, and its digits rule keeps counting superscripts (`м²`). Turkish had no test dictionary, so its two dictionary tests returned early wherever none was installed, CI included; it has a synthetic one now, also in the dictionaries-folder test. Tests: proofing_options_reach_the_russian_check and proofing_options_reach_the_turkish_check. Calling the old checks from lib.rs fails each, and so does ungating the Russian part rule or the Turkish hyphen recursion. --- crates/proof/src/lib.rs | 9 ++++--- crates/proof/src/ru/mod.rs | 45 +++++++++++++++++++++++++++----- crates/proof/src/tr/mod.rs | 39 ++++++++++++++++++++++----- crates/proof/src/tr/suggest.rs | 5 +--- crates/proof/src/tr/test_dict.rs | 22 ++++++++++++++++ 5 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 crates/proof/src/tr/test_dict.rs diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 8d0baba5e..47216d72a 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -778,8 +778,8 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts), ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts), ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts), - ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word), - ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), + ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct_with_options(word, opts), + ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct_with_options(word, opts), ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => { accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts) } @@ -1075,6 +1075,7 @@ mod tests { ("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC), ("it_IT", it::test_dict::AFF, it::test_dict::DIC), ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), + ("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC), ("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC), ("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC), ("id_ID", id::test_dict::AFF, id::test_dict::DIC), @@ -1094,9 +1095,9 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it. + // Turkish follows the four below; Russian, Czech, Indonesian and the Spanishes follow it. let found = installed_dictionaries(); - assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr)); + assert_eq!(found.get(4), Some(&(ProofLang::Tr, Some(dir.join("tr_TR.dic"))))); assert_eq!( found[7..], [ diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index c92e27125..d5035e6a3 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -130,18 +130,30 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits or superscripts. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary (an empty one accepts everything). pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // no Russian dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'); let w = w.trim_matches('-'); let n = w.chars().count(); - if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -161,7 +173,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { return true; } if w.contains('-') { - return hyphenated(d, w); + return hyphenated(d, w, opts); } solid_compound(d, w) } @@ -340,7 +352,7 @@ fn known(d: &AffixDict, w: &str) -> bool { /// A hyphenated word that isn't in the dictionary whole: its parts, with the particles, /// prefixes and linking vowels Russian joins with a hyphen. -fn hyphenated(d: &AffixDict, w: &str) -> bool { +fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let parts: Vec<&str> = w.split('-').collect(); if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 { return false; @@ -353,7 +365,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { && lp.iter().skip(k).all(|p| PARTICLES.contains(p)) { let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default(); - return known(d, &base) || (base.contains('-') && hyphenated(d, &base)); + return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts)); } match lp.as_slice() { // `кое-кто`, `кой-какой`. @@ -369,7 +381,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -497,4 +509,25 @@ mod tests { crate::add_word("Зкщфрусскоеслово"); assert!(is_correct("зкщфрусскоеслово")); } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_russian_check() { + test_dict::install(); + let lang = crate::ProofLang::Ru; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default"); + assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults)); + assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too"); + assert!(crate::is_correct_in_with("превет2", lang, &defaults)); + assert!(!crate::is_correct_in_with("превет2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index bcab1546b..ab3fe92d7 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -15,6 +15,8 @@ pub mod dict; pub mod grammar; pub mod hyphen; pub mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; use std::sync::{Arc, RwLock}; @@ -87,15 +89,26 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain +/// numbers off, words with digits are checked against the dictionary too. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary. pub fn is_correct_with(d: &TrDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // No Turkish dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); let w = w.trim_matches(['\'', '’']); let n = w.chars().count(); - if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return true; } // Check user dictionary first @@ -120,7 +133,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool { } // Compound words separated by hyphens (e.g., `afyon-karahisar`) - if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) { + if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) { return true; } @@ -146,11 +159,8 @@ mod tests { } #[test] - fn installed_dictionary_checks_words_if_present() { - // Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check. - if !has_dictionary() { - return; - } + fn installed_dictionary_checks_words() { + test_dict::install(); assert!(is_correct("kitap")); assert!(is_correct("kitaplar")); assert!(is_correct("evde")); @@ -165,4 +175,19 @@ mod tests { crate::add_word("benimbilmedigimkelime"); assert!(is_correct("benimbilmedigimkelime")); } + + /// With Ignore words that contain numbers off, Turkish checks words with digits against the + /// dictionary instead of accepting them, hyphen parts included. + #[test] + fn proofing_options_reach_the_turkish_check() { + test_dict::install(); + let lang = crate::ProofLang::Tr; + let defaults = crate::Options::default(); + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("kitapp2", lang, &defaults)); + assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("kitapp2"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index dedc1bb26..f91f8e3c9 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -285,10 +285,7 @@ mod tests { #[test] fn typo_transposition_is_suggested() { - // Suggestions come from the installed Turkish dictionary; without one there are none. - if !super::super::has_dictionary() { - return; - } + crate::tr::test_dict::install(); let sugs = suggest("sorna", 3); assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}"); diff --git a/crates/proof/src/tr/test_dict.rs b/crates/proof/src/tr/test_dict.rs new file mode 100644 index 000000000..de4a13195 --- /dev/null +++ b/crates/proof/src/tr/test_dict.rs @@ -0,0 +1,22 @@ +//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// No affix rules: the forms the tests look up are listed whole. +pub const AFF: &str = "SET UTF-8\n"; + +/// The words the Turkish tests expect to be known. +pub const DIC: &str = "7 +kitap +kitaplar +ev +evde +olacaklardan +sonra +geldin +"; + +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} From 281392ac154286573ba283ee3456411fb8f97540 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:21:57 -0100 Subject: [PATCH 02/21] Croatian, Italian and Russian suggestions skip NOSUGGEST words #1048 (6e25d482c) taught affix.rs about NOSUGGEST, the dictionary flag for words that are correct but never offered (vulgarisms, rare forms), and patched the Polish and Portuguese copies of the suggestion engine; Czech's started with it. The Croatian, Italian and Russian copies never got it: they test candidates with `contains`, so a suggestion could be a word the dictionary says not to offer. They now use `contains_suggestible` and drop a suggestion with any part `can_suggest` refuses, as the others do. Tests: Croatian and Italian run the shared assert_nosuggest_is_honoured. Russian gets its own, since its MAP and REP tables are built in and the shared Latin-script check doesn't reach them. Each fails on the old code. --- crates/proof/src/hr/suggest.rs | 11 +++++++++-- crates/proof/src/it/suggest.rs | 12 ++++++++++-- crates/proof/src/ru/suggest.rs | 23 +++++++++++++++++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index 875bf98c7..d6f8159ac 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -52,11 +52,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -69,6 +69,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -348,6 +351,10 @@ fn scan(col: &mut Collector, chars: &[char]) { mod tests { use super::*; #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] { diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 500d1c09d..6a043ff2b 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -73,11 +73,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -90,6 +90,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -371,6 +374,11 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn italian_misspellings_get_the_right_suggestion() { crate::it::test_dict::install(); diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index cb4c3791d..24eeb7642 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -161,11 +161,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -178,6 +178,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -458,6 +461,22 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never + /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the + /// shared Latin-script check doesn't reach it. + #[test] + fn nosuggest_words_are_never_offered() { + let offered = |aff: &str, input: &str| { + let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap(); + suggest_in(&d, input, 16) + }; + assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared"); + for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] { + let s = offered("SET UTF-8\nNOSUGGEST N\n", input); + assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}"); + } + } + #[test] fn russian_misspellings_get_the_right_suggestion() { crate::ru::test_dict::install(); From d2f45ddd966f896090fd840246c280fb4cced2c7 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:33:17 -0100 Subject: [PATCH 03/21] Proofing: one dictionary slot for every language Nine languages each kept their own copy of the same code: the installed dictionary behind a lock, read on first use and logged, replaced by install_dictionary, forgotten by reload_dictionary, and its file reported by installed_path. The copies had drifted already. Every one documented that a dictionary given with install_dictionary has no file and isn't reported, but only Russian and Czech did that (#3252); the other seven reported the disk file anyway. `DictSlot` (proof/src/slot.rs) is that code once. It is generic over the dictionary type, so Turkish's `TrDict` fits, and keeps the dictionary and its in-memory origin under one lock as #3252 made it. Each language keeps its functions as one-line wrappers over its slot, so no caller changes, and the log lines are the same. installed_path now follows the documented rule in every language. Nothing outside tests installs a dictionary in memory, but tests do, concurrently, so the dictionaries-folder tests in proof and engine could race on the reported path of a language another test installs. They now accept the folder's file or none there, and check that the folder is searched first with hunspell::find, which no install affects. Tests: proof 279, engine 927, layout 501 (lib); slot.rs has a test of its own. --- crates/engine/src/tests.rs | 18 +++-- crates/proof/src/cs/mod.rs | 60 +++------------ crates/proof/src/es/mod.rs | 71 ++++++----------- crates/proof/src/hr/mod.rs | 45 +++-------- crates/proof/src/id/mod.rs | 50 ++++-------- crates/proof/src/it/mod.rs | 44 +++-------- crates/proof/src/lib.rs | 50 ++++++------ crates/proof/src/pl/mod.rs | 52 ++++--------- crates/proof/src/pt/mod.rs | 44 +++-------- crates/proof/src/ru/mod.rs | 68 ++++------------ crates/proof/src/slot.rs | 150 ++++++++++++++++++++++++++++++++++++ crates/proof/src/tr/dict.rs | 15 ++++ crates/proof/src/tr/mod.rs | 57 ++++---------- 13 files changed, 338 insertions(+), 386 deletions(-) create mode 100644 crates/proof/src/slot.rs diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 7c6ad50e7..616d95557 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -7056,14 +7056,22 @@ fn dictionaries_folder_option_is_saved_searched_and_reported() { assert_eq!(r["dictionariesFolder"], json!(dir.to_string_lossy())); let dicts = r["dictionaries"].as_array().unwrap(); let file = |lang: &str| dicts.iter().find(|d| d["language"] == lang).map(|d| d["file"].clone()); - assert_eq!(file("pl"), Some(json!(dir.join("pl_PL.dic").to_string_lossy()))); - assert_eq!(file("pt"), Some(json!(dir.join("pt_BR.dic").to_string_lossy()))); - assert_eq!(file("id"), Some(json!(dir.join("id_ID.dic").to_string_lossy()))); + // A language reports the folder's file, or none while a dictionary another test gave with + // install_dictionary() is in use (those tests run concurrently); the folder is searched first + // either way. + let reported = |got: Option<&serde_json::Value>, name: &str| { + let want = dir.join(format!("{name}.dic")); + assert!(got.is_some_and(|f| f.is_null() || *f == json!(want.to_string_lossy())), "{name}: {dicts:?}"); + assert_eq!(wordcraft_proof::hunspell::find(&[name]).map(|(_, dic)| dic), Some(want)); + }; + reported(file("pl").as_ref(), "pl_PL"); + reported(file("pt").as_ref(), "pt_BR"); + reported(file("id").as_ref(), "id_ID"); // Each Spanish variety reports its own file (Spain, Mexico, Latin America, Uruguay, in order). let spanish: Vec = dicts.iter().filter(|d| d["language"] == "es").map(|d| d["file"].clone()).collect(); assert_eq!(spanish.len(), 4); - assert_eq!(spanish.first(), Some(&json!(dir.join("es_ES.dic").to_string_lossy()))); - assert_eq!(spanish.last(), Some(&json!(dir.join("es_UY.dic").to_string_lossy()))); + reported(spanish.first(), "es_ES"); + reported(spanish.last(), "es_UY"); // Polish text is checked against the folder's dictionary. run(&mut s, "document.setText", json!({"text": "rzeka płynie żeka"})); run(&mut s, "select.all", json!({})); diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index fee7c561c..55e21aef6 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -7,83 +7,47 @@ pub(crate) mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Czech Hunspell dictionary. const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; +/// The Czech dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); + /// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Czech spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; + DICT.reload(); } -/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. +/// The `.dic` file of the installed Czech dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 94c6511ab..253642c83 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -16,16 +16,19 @@ mod suggest; pub(crate) mod test_dict; pub mod voseo; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::ProofLang; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first -/// use, then the installed one (or an empty one). -static DICTS: [RwLock>>; 4] = [const { RwLock::new(None) }; 4]; +/// The dictionary of each variety, found on first use (see [`DictSlot`]). +static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); /// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the /// generic one RLA-ES publishes). @@ -40,79 +43,53 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e /// or generic one. const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"]; -/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's -/// file names; Spain for any other language. -fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) { +/// `lang`'s variety's dictionary; Spain's for any other language. +fn slot(lang: ProofLang) -> &'static DictSlot { match lang { - ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO), - ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA), - ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY), - _ => (0, "Spanish (Spain)", SPAIN), + ProofLang::EsMx => &MEXICO_DICT, + ProofLang::Es419 => &LATIN_AMERICA_DICT, + ProofLang::EsUy => &URUGUAY_DICT, + _ => &SPAIN_DICT, } } /// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`, /// `es_UY`): the first of its file names, used for the status-bar hint and load errors. pub fn dictionary_name(lang: ProofLang) -> &'static str { - variety(lang).2.first().copied().unwrap_or("es_ES") + slot(lang).name() } /// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given /// with [`install_dictionary`], or an empty one when none is available (spelling is off then; /// see [`has_dictionary`]). pub fn dictionary(lang: ProofLang) -> Arc { - let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) }; - if let Some(d) = slot.read().unwrap_or_else(|e| e.into_inner()).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed(lang)); - let mut slot = slot.write().unwrap_or_else(|e| e.into_inner()); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed(lang: ProofLang) -> AffixDict { - let (_, name, files) = variety(lang); - let key = dictionary_name(lang); - let Some((aff, dic)) = crate::hunspell::find(files) else { - crate::hunspell::clear_load_error(key); - log::info!("{name} spelling: no Hunspell {key} dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("{name} spelling: {e}"); - AffixDict::default() - }) + slot(lang).get() } /// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text, /// decoded to UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - if let Some(slot) = DICTS.get(variety(lang).0) { - *slot.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); - } - Ok(()) + slot(lang).install(aff, dic) } /// Forget every variety's loaded dictionary, so the next check looks for an installed one again /// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - for slot in &DICTS { - *slot.write().unwrap_or_else(|e| e.into_inner()) = None; + for slot in [&SPAIN_DICT, &MEXICO_DICT, &LATIN_AMERICA_DICT, &URUGUAY_DICT] { + slot.reload(); } } /// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder -/// or the system locations now (File › Options › Proofing shows it); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// or the system locations now (File › Options › Proofing shows it); `None` when there is none, +/// or when one given with [`install_dictionary`] is in use. pub fn installed_path(lang: ProofLang) -> Option { - crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic) + slot(lang).installed_path() } /// Is a dictionary available for `lang`'s variety (spelling is checked)? pub fn has_dictionary(lang: ProofLang) -> bool { - !dictionary(lang).is_empty() + slot(lang).has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? @@ -200,7 +177,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); for lang in VARIETIES { - let found = crate::hunspell::find_in(&dir, variety(lang).2); + let found = crate::hunspell::find_in(&dir, slot(lang).names()); assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}"); if let Some((aff, dic)) = found { assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang)); @@ -215,7 +192,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); } for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] { - assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); + assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); assert_eq!(dictionary_name(lang), name); } std::fs::remove_dir_all(dir).unwrap(); diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index ee9d58e92..6372a47d2 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -9,68 +9,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Croatian Hunspell dictionary. const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; +/// The Croatian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); + /// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed"); - return AffixDict::default(); - }; - - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Croatian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Croatian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Croatian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)? diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 012babc0e..65b820dfc 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -9,74 +9,54 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by /// Java locales). const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; -/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number -/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. -const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; +/// The Indonesian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); /// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Indonesian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it); `None` when there is none. A -/// dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Indonesian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is an Indonesian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } +/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number +/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. +const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; + /// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)? fn is_latin_letter(c: char) -> bool { c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 027006a69..168883e17 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -18,67 +18,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Italian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Italian Hunspell dictionary (Italy first, then Switzerland). const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; +/// The Italian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); + /// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Italian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Italian spelling: no Hunspell it_IT dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Italian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Italian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is an Italian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)? diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 47216d72a..7469c4d9b 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -65,6 +65,7 @@ pub mod patterns; pub mod pl; pub mod pt; pub mod ru; +mod slot; pub mod tr; pub mod vi; @@ -1095,36 +1096,35 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish follows the four below; Russian, Czech, Indonesian and the Spanishes follow it. + // Every language is listed, Russian and Czech too (#3226: they were once missing, so + // neither showed up in File › Options › Proofing). A language reports the folder's file, + // or none while a dictionary given with install_dictionary() is in use (#3252 review); + // the languages' own tests install theirs concurrently, so an exact path would race them. + // The folder is searched first whatever is in use. let found = installed_dictionaries(); - assert_eq!(found.get(4), Some(&(ProofLang::Tr, Some(dir.join("tr_TR.dic"))))); + let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"]; assert_eq!( - found[7..], + found.iter().map(|(lang, _)| *lang).collect::>(), [ - (ProofLang::Id, Some(dir.join("id_ID.dic"))), - (ProofLang::Es, Some(dir.join("es_ES.dic"))), - (ProofLang::EsMx, Some(dir.join("es_MX.dic"))), - (ProofLang::Es419, Some(dir.join("es_419.dic"))), - (ProofLang::EsUy, Some(dir.join("es_UY.dic"))), + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy ] ); - assert_eq!( - found[..4], - [ - (ProofLang::Pl, Some(dir.join("pl_PL.dic"))), - (ProofLang::Pt, Some(dir.join("pt_BR.dic"))), - (ProofLang::It, Some(dir.join("it_IT.dic"))), - (ProofLang::Hr, Some(dir.join("hr_HR.dic"))), - ] - ); - // #3226: Russian and Czech were missing from installed_dictionaries() entirely (no - // installed_path() in either module), so neither ever showed up in File › Options › - // Proofing, installed or not. Their languages, not their exact path, is what belongs - // here: unlike the four above, installed_path() for Russian and Czech also reflects - // whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own - // test modules do, concurrently, in tests of their own — asserting an exact path here - // would race them for no reason this test needs to care about. - assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}"); + for ((lang, path), name) in found.iter().zip(names) { + let file = dir.join(format!("{name}.dic")); + assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}"); + assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}"); + } assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy)); assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 18c968feb..4d79986d9 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -17,67 +17,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Polish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Polish Hunspell dictionary. const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; +/// The Polish dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); + /// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Polish spelling: no Hunspell pl_PL dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Polish spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Polish dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Polish dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index de4198259..860cf4579 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -16,67 +16,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Portuguese Hunspell dictionary. const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; +/// The Portuguese dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); + /// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Portuguese spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Portuguese dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Portuguese dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)? diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index d5035e6a3..e46bf305f 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -28,83 +28,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Russian Hunspell dictionary. const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; +/// The Russian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); + /// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Russian spelling: no Hunspell ru_RU dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Russian spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; + DICT.reload(); } -/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. +/// The `.dic` file of the installed Russian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)? diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs new file mode 100644 index 000000000..3c0f2bd35 --- /dev/null +++ b/crates/proof/src/slot.rs @@ -0,0 +1,150 @@ +//! The installed Hunspell dictionary of a proofing language, in one shape for every language: +//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`]. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::affix::AffixDict; + +/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own. +pub(crate) trait Dictionary: Default + Send + Sync { + /// The installed `.aff` and `.dic` files, read. + fn load(aff: &Path, dic: &Path) -> Result; + /// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed. + fn parse(aff: &str, dic: &str) -> Result; + /// No words: the language has no spelling marks. + fn is_empty(&self) -> bool; +} + +impl Dictionary for AffixDict { + fn load(aff: &Path, dic: &Path) -> Result { + crate::hunspell::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + AffixDict::from_hunspell(aff, dic, &[]) + } + + fn is_empty(&self) -> bool { + AffixDict::is_empty(self) + } +} + +/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on +/// disk (it then has no file to report, even if a matching file also happens to exist): one lock, +/// so a reload and an install can't interleave their two fields. +struct State { + dict: Option>, + in_memory: bool, +} + +/// One language's dictionary. +pub(crate) struct DictSlot { + /// File names the dictionary may have, the usual one first (also the key of its load error). + names: &'static [&'static str], + /// The language's English name, for the log. + english: &'static str, + state: RwLock>, +} + +impl DictSlot { + pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false }) } + } + + fn read(&self) -> RwLockReadGuard<'_, State> { + self.state.read().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self) -> RwLockWriteGuard<'_, State> { + self.state.write().unwrap_or_else(PoisonError::into_inner) + } + + /// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names. + pub(crate) fn name(&self) -> &'static str { + self.names.first().copied().unwrap_or_default() + } + + /// The file names the dictionary may have, the usual one first. + #[cfg(test)] + pub(crate) fn names(&self) -> &'static [&'static str] { + self.names + } + + /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or + /// an empty one when none is available (the language's spelling is off then). + pub(crate) fn get(&self) -> Arc { + if let Some(d) = self.read().dict.as_ref() { + return d.clone(); + } + let loaded = Arc::new(self.load_installed()); + // Another thread (or an install) may have filled it meanwhile: that one wins. + self.write().dict.get_or_insert(loaded).clone() + } + + fn load_installed(&self) -> D { + let name = self.name(); + let Some((aff, dic)) = crate::hunspell::find(self.names) else { + crate::hunspell::clear_load_error(name); + log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english); + return D::default(); + }; + crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("{} spelling: {e}", self.english); + D::default() + }) + } + + /// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for + /// example one the user picked. On error the current dictionary stays. + pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> { + let d = D::parse(aff, dic)?; + let mut slot = self.write(); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; + Ok(()) + } + + /// Forget the dictionary, so the next check looks for an installed one again (after + /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). + pub(crate) fn reload(&self) { + let mut slot = self.write(); + slot.dict = None; + slot.in_memory = false; + } + + /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the + /// system locations now (File › Options › Proofing shows it); `None` when there is none, or + /// when the dictionary in use was given with [`Self::install`] and so has no file. + pub(crate) fn installed_path(&self) -> Option { + if self.read().in_memory { + return None; + } + crate::hunspell::find(self.names).map(|(_, dic)| dic) + } + + /// Is a dictionary available (spelling is checked)? + pub(crate) fn has_dictionary(&self) -> bool { + !self.get().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static SLOT: DictSlot = DictSlot::new(&["zz_SLOT_TEST"], "Test"); + + #[test] + fn an_installed_dictionary_has_no_file_until_reloaded() { + assert!(!SLOT.has_dictionary(), "nothing installed under this name"); + assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(!SLOT.has_dictionary(), "a refused install keeps the current one"); + SLOT.install("", "1\nkot\n").unwrap(); + assert!(SLOT.has_dictionary() && SLOT.get().check("kot")); + assert_eq!(SLOT.installed_path(), None); + SLOT.reload(); + assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); + assert_eq!(SLOT.name(), "zz_SLOT_TEST"); + } +} diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index ed57cbca7..b47ba7c1e 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -26,6 +26,21 @@ pub struct TrDict { max_suffix_len: usize, } +impl crate::slot::Dictionary for TrDict { + fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let read = |p: &std::path::Path| std::fs::read_to_string(p).map_err(|e| e.to_string()); + TrDict::from_hunspell(&read(aff)?, &read(dic)?) + } + + fn parse(aff: &str, dic: &str) -> Result { + TrDict::from_hunspell(aff, dic) + } + + fn is_empty(&self) -> bool { + TrDict::is_empty(self) + } +} + impl TrDict { /// Returns true if empty (no dictionary loaded). pub fn is_empty(&self) -> bool { diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index ab3fe92d7..eb5f0ea52 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -18,70 +18,45 @@ pub mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; + +use crate::slot::DictSlot; pub use dict::TrDict; pub use suggest::suggest; -/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Turkish Hunspell dictionary. const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; +/// The Turkish dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); + /// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available. pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> TrDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); - return TrDict::default(); - }; - let read = |p: &std::path::Path| std::fs::read_to_string(p); - match (read(&aff), read(&dic)) { - (Ok(aff_text), Ok(dic_text)) => crate::hunspell::note_load(NAMES[0], &dic, TrDict::from_hunspell(&aff_text, &dic_text)).unwrap_or_else(|e| { - log::warn!("Turkish spelling: {e}"); - TrDict::default() - }), - (a, d) => { - let reason = a.err().or(d.err()).map_or_else(String::new, |e| e.to_string()); - let _ = crate::hunspell::note_load::<()>(NAMES[0], &dic, Err(reason)); - TrDict::default() - } - } -} - -/// Use this Turkish dictionary from now on. +/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the +/// current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = TrDict::from_hunspell(aff, dic)?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } -/// Reload the dictionary on next check. +/// Forget the loaded dictionary, so the next check looks for an installed one again. pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Turkish dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Turkish dictionary available? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `word` spelled correctly in Turkish? From 577d5d61f876e4c7bc9949b1b0a96e4654874cdc Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:48:47 -0100 Subject: [PATCH 04/21] Proof tests: the user-dictionary file test waits for the dictionary tests user_dictionary_persists_through_a_file saves the whole user dictionary to a file and loads it back. Run beside user_dictionary_and_ignore_all_match_both_normalization_forms, it could catch a word that test had added and bring it back after the test removed it, so the other test's last check failed now and then (once in about two dozen full runs here). It now holds DICTIONARY_TESTS, as the other test does. --- crates/proof/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 7469c4d9b..ad516b295 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1059,7 +1059,9 @@ mod tests { use super::*; - /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test. + /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land + /// mid-test, and by those that save the whole user dictionary and load it back, which would + /// bring back a word another test just removed. static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// #1063: the dictionaries folder is searched before the system locations; the dictionaries @@ -1579,6 +1581,7 @@ mod tests { /// #40: the user dictionary survives a save and a fresh load. #[test] fn user_dictionary_persists_through_a_file() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist")); let _ = std::fs::remove_dir_all(&dir); // The folder is created on the first save. From e7eda2aff0462145bbb325a3d5c1f674ab1715fb Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:49:22 -0100 Subject: [PATCH 05/21] Proofing: one suggestion engine with per-language tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Czech's dictionary-driven ranking (case, MAP, REP, missing spaces, KEY, TRY edits, a stem scan) had five older copies. Polish, Portuguese, Croatian, Italian and Russian each kept one: 300 to 400 lines of the same rounds with a few lines of language knowledge mixed in. Fixes reached some copies only. NOSUGGEST (#1048) and the first-difference tie-break (#797) never got to all of them, and none had the KEY round or the bound on TRY letters. affix_suggest.rs is that ranking once. A language's `Tuning` holds what is its own: the short words that run into their neighbours (Polish, Portuguese, Croatian, Italian), Russian's split rule, the letter groups and confusions its dictionaries lack (Croatian, Russian), letters to try for a dictionary without a TRY line, and cost hooks for Polish final devoicing and Russian е/ё. Czech, Indonesian and Spanish use the dictionary's tables alone. Turkish keeps its own suggester, which reads another dictionary type with rounds of its own. The language tables add to the dictionary's instead of replacing them, and TRY letters are tried in the dictionary's order, commonest first, as the copies did (the Czech engine sorted them). Old and new compared on the test dictionaries, with every one-letter typo of each word plus the tests' misspellings, top eight suggestions: Czech, Croatian, Italian and Portuguese give the same for all 7,903 inputs. Polish differs for 44 of 3,244 and Russian for 33 of 2,346, all orderings among candidates of equal cost, from the tie-break they now share (`ie` → `nie` before `i`). New tests pin the moved knowledge: the tuning mechanism, Croatian's groups without MAP lines, and Russian's costs and split rule. Turning each knob off fails one. Proof: 283 tests. --- crates/proof/src/affix_suggest.rs | 493 ++++++++++++++++++++++++++++++ crates/proof/src/cs/suggest.rs | 358 +--------------------- crates/proof/src/es/suggest.rs | 8 +- crates/proof/src/hr/suggest.rs | 351 ++------------------- crates/proof/src/id/suggest.rs | 9 +- crates/proof/src/it/suggest.rs | 350 +-------------------- crates/proof/src/lib.rs | 1 + crates/proof/src/pl/suggest.rs | 361 ++-------------------- crates/proof/src/pt/suggest.rs | 356 +-------------------- crates/proof/src/ru/suggest.rs | 391 ++++-------------------- 10 files changed, 625 insertions(+), 2053 deletions(-) create mode 100644 crates/proof/src/affix_suggest.rs diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs new file mode 100644 index 000000000..1da355595 --- /dev/null +++ b/crates/proof/src/affix_suggest.rs @@ -0,0 +1,493 @@ +//! Spelling suggestions from an affix dictionary, one engine for every language that reads one +//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come +//! in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`): +//! adding a missing accent is the commonest slip and ranks first, while taking away one the +//! writer typed ranks after the known confusions; +//! 3. the REP table of common confusions (`żeka` → `rzeka`); +//! 4. a missing space (`napewno` → `na pewno`); +//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with +//! the next or mistyped (TRY letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in +//! which a MAP slip costs half (two typos at most). +//! +//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may +//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every +//! round, so a call costs a few thousand dictionary lookups at most. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters. +pub(crate) const MAX_LETTERS: usize = 40; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +pub(crate) const COST_ADD_DIACRITIC: u32 = 4; +pub(crate) const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +pub(crate) const COST_SPLIT: u32 = 12; +/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]). +const COST_SPLIT_SHORT: u32 = 17; +/// A missing space between two longer halves with no short word beside it ([`Split::Near`]). +pub(crate) const COST_SPLIT_LONG: u32 = 18; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +pub(crate) const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +/// What a language adds to its dictionary's tables. +pub(crate) struct Tuning { + /// Where a missing space may go (round 4). + pub split: Split, + /// TRY letters for a dictionary without a TRY line. + pub alphabet: &'static str, + /// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`). + pub map: &'static [&'static str], + /// REP confusions the language adds to its dictionary's. + pub rep: &'static [(&'static str, &'static str)], + /// The cost of a REP confusion. + pub rep_cost: u32, + /// Most letters a MAP variant changes. + pub max_map_changes: usize, + /// The cost of changing letter `from` to its MAP relative `to`. + pub map_cost: fn(char, char) -> u32, + /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. + pub mistype_cost: fn(&[char], usize, char) -> u32, +} + +/// Where a missing space may go. +pub(crate) enum Split { + /// Anywhere, dearer when a half is shorter than three letters. + Anywhere, + /// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one + /// that runs into the word before (`kocham cię`), dearer between two longer halves, and never + /// leaving a stray short half (`że ka`). + Near { leading: &'static [&'static str], trailing: &'static [&'static str] }, + /// The language's own rule: the cost of the halves `a b`, or `None` to skip them. + Custom(fn(&str, &str) -> Option), +} + +impl Tuning { + /// The dictionary's tables alone (Czech, Indonesian, Spanish). + pub(crate) const DEFAULT: Tuning = + Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type is likelier right, +/// so taking one away costs more than a known confusion from the REP table. +pub(crate) fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Any mistyped letter costs the same. +fn mistype_cost(_: &[char], _: usize, _: char) -> u32 { + COST_MISTYPE +} + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + tested: HashSet, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + return; + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP groups: the dictionary's, then the language's. +fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec]> { + if t.map.is_empty() { + return Cow::Borrowed(d.map()); + } + Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect()) +} + +/// The other members of every MAP group of `c` (its likely confusions), each once. +fn alternatives(c: char, map: &[Vec]) -> Vec { + let mut out: Vec = Vec::new(); + for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) { + if x != c && !out.contains(&x) { + out.push(x); + } + } + out +} + +/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, +/// letter by letter), with its cost, each written after `buf`. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some((&c, rest)) = chars.split_first() else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); + buf.push(c); + map_variants(rest, later, t, changes, cost, buf, out); + buf.pop(); + if changes < t.max_map_changes { + for &a in here { + buf.push(a); + map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`. +pub(crate) fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered. +fn split_cost(split: &Split, a: &str, b: &str) -> Option { + let long = |part: &str| part.chars().count() >= 3; + match split { + Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }), + Split::Near { leading, trailing } => { + if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } + } + Split::Custom(cost) => cost(a, b), + } +} + +/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case. +pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + // 1. Letter case. + if d.contains(&input) && word != input { + col.keep(input.clone(), COST_CASE); + } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { + col.keep(capitalize(&input), COST_CASE); + } + // 2. Accents and other MAP slips. + let map = groups(d, t); + let alts: Vec> = chars.iter().map(|&c| alternatives(c, &map)).collect(); + let mut variants = Vec::new(); + map_variants(&chars, &alts, t, 0, 0, &mut Vec::with_capacity(n), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } + // 3. Common confusions. + let rep = d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } + // 4. A missing space; both halves must be dictionary words. + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(&t.split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } + // 5. Keyboard neighbours from the dictionary's KEY rows. + for row in d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } + // One typo, with the TRY letters (the language's when the dictionary has none), bounded and + // each once, in their order: a TRY line lists the commonest letters first, so equal-cost + // candidates come out in that order. + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.clone(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.clone(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.clone(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(&chars, i, c)); + } + } + } + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars, &map); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let t = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text, + }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { + let d = col.d; + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + #[test] + fn distance_weighs_map_slips_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY + /// line, and a split rule decides which missing spaces are offered. + #[test] + fn tuning_adds_to_the_dictionary_tables() { + let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap(); + let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT }; + assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into())); + assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group"); + assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair"); + assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters"); + let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT }; + assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into())); + assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half"); + assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into())); + } +} diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 4e5ecd17c..05eba74a8 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -1,168 +1,8 @@ -//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits -//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian -//! ([`crate::id`]) uses the same ranking with its own dictionary. +//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed +//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Longest word suggestions are looked for, in letters (Spanish shares the bound). -pub(crate) const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 17; -const COST_KEY: u32 = 13; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (a capitalized name). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (typing without diacritics); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; /// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -171,201 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space; both halves must be dictionary words. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |part: &str| part.chars().count() >= 3; - let cost = if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_RARE }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. Keyboard neighbours from the installed dictionary's KEY rows. - for row in d.key().split('|').take(32) { - let keys: Vec = row.chars().take(128).collect(); - for pair in keys.windows(2) { - let [a, b] = pair else { continue }; - for (i, &c) in chars.iter().enumerate() { - let replacement = if c == *a { - Some(*b) - } else if c == *b { - Some(*a) - } else { - None - }; - if let Some(replacement) = replacement { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = replacement; - } - col.offer(&v.iter().collect::(), COST_KEY); - } - } - } - } - // One-letter edits use only the runtime TRY alphabet, with bounded work. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).take(128).collect(); - letters.sort_unstable(); - letters.dedup(); - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (the first changed letter breaks ties). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) } #[cfg(test)] mod tests { use super::*; - #[test] - fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { - let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); - let suggestions = suggest_in(&d, "počítačz", 8); - assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); - assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); - } - #[test] fn suggestions_are_fast_enough() { crate::cs::test_dict::install(); diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs index b42efa2ee..0568ae870 100644 --- a/crates/proof/src/es/suggest.rs +++ b/crates/proof/src/es/suggest.rs @@ -1,5 +1,5 @@ -//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to +//! Spanish suggestions. The ranking is the shared dictionary-driven one +//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to //! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, //! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. //! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first @@ -7,7 +7,7 @@ use crate::ProofLang; use crate::affix::{AffixDict, Casing, capitalize, casing}; -use crate::cs::suggest::MAX_LETTERS; +use crate::affix_suggest::{MAX_LETTERS, Tuning}; /// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's /// case. @@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec _ => v, }); } - for s in crate::cs::suggest::suggest_in(d, word, max) { + for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) { if !out.contains(&s) { out.push(s); } diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index d6f8159ac..d8c19b99b 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -1,12 +1,11 @@ -//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance, -//! missing spaces and a bounded search. Proper Croatian corrections depend on -//! the actual installed dictionary entries. +//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of +//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines. -use std::collections::{HashMap, HashSet}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Short words often run into the next one (`não é`, `da água`, `para isso`). +/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`). const LEADING_WORDS: &[&str] = &[ "a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će", "bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad", @@ -14,162 +13,13 @@ const LEADING_WORDS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Croatian adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiourntsjkvlmdpbgczčćšžđfh", + map: &["cčć", "dđ", "sš", "zž"], + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -178,173 +28,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']]; - let map = if d.map().is_empty() { &fallback_map } else { d.map() }; - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -354,6 +38,13 @@ mod tests { fn nosuggest_stems_and_forms_are_never_offered() { crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); } + /// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća` + /// first, before a typo that costs more (`kuha`). + #[test] + fn diacritics_without_map_lines() { + let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap(); + assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]); + } #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); diff --git a/crates/proof/src/id/suggest.rs b/crates/proof/src/id/suggest.rs index 562af9a82..b42efde98 100644 --- a/crates/proof/src/id/suggest.rs +++ b/crates/proof/src/id/suggest.rs @@ -1,7 +1,6 @@ -//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem -//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is -//! hard-coded here. +//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing +//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID` +//! dictionary, so nothing Indonesian is hard-coded here. use crate::affix::AffixDict; @@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - crate::cs::suggest::suggest_in(d, word, max) + crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT) } #[cfg(test)] diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 6a043ff2b..55f869091 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -1,25 +1,9 @@ -//! Italian spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`roma` → `Roma`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`, -//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`); -//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`per me`, `a Milano`, `di Roma`). const LEADING_WORDS: &[&str] = &[ @@ -35,161 +19,12 @@ const TRAILING_WORDS: &[&str] = &[ "mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove", "è", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; +/// What Italian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `à` → `a`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} /// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -198,176 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent. - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index ad516b295..4bf4da78b 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -50,6 +50,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod affix; +mod affix_suggest; pub mod bg; pub mod cs; pub mod dict; diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs index 0e32a1eaf..7baeaa54f 100644 --- a/crates/proof/src/pl/suggest.rs +++ b/crates/proof/src/pl/suggest.rs @@ -1,26 +1,9 @@ -//! Polish spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`warszawa` → `Warszawa`); -//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy` -//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most -//! common slip, so adding them ranks first, while taking away a diacritic the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` → -//! `huśtawka`, `pszyjaciel` → `przyjaciel`); -//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham -//! cię`), cheaper when one half is a short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which a diacritic slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand -//! dictionary lookups at most. +//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`), +//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning}; /// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede /// wszystkim`). @@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[ /// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). const TRAILING_WORDS: &[&str] = &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; +/// What Polish adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx", + mistype_cost, + ..Tuning::DEFAULT +}; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moze` → `może`, then - // `Mozę`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the -/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the -/// writer did type is likelier right, so taking one away costs more than a known confusion from -/// the REP table (`żeka` → `rzeka` before `zeka`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) +/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it +/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo. +fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 { + let last = i + 1 == chars.len(); + if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE } } /// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. @@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Diacritics and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half (`że ka`). - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - // Final devoicing can make a voiced consonant sound voiceless. - let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g')); - let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }; - col.offer(&s(&v), cost); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs index 36f1c7b07..f8f33bff6 100644 --- a/crates/proof/src/pt/suggest.rs +++ b/crates/proof/src/pt/suggest.rs @@ -1,26 +1,9 @@ -//! Portuguese spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`brasília` → `Brasília`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`, -//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` → -//! `exceção`, `previlégio` → `privilégio`); -//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`não é`, `da água`, `para isso`). const LEADING_WORDS: &[&str] = &[ @@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[ "me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si", "eu", "tu", "que", "de", "em", "é", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Portuguese adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw", + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index 24eeb7642..135f4152f 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -1,30 +1,14 @@ -//! Russian spelling suggestions, best first. -//! -//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the -//! common confusions are defined here. Candidates come in rounds of rising cost, each checked -//! against the dictionary: -//! 1. letter case (`москва` → `Москва`); -//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`, -//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to -//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`); -//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or -//! doubled consonant, `здраствуй` → `здравствуй`); -//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped; -//! 6. when that finds fewer than asked for: the stems that start like the word and share the -//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within -//! a letter group costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round. +//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP` +//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` → +//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule +//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning}; /// Letters confused with each other: a slip inside a group is cheaper than another typo. const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"]; - /// Common confusions: a written sequence and what was meant. const REP: &[(&str, &str)] = &[ ("жы", "жи"), @@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[ ("ц", "тс"), ("тс", "ц"), ]; - /// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`). const LEADING_WORDS: &[&str] = &[ "не", @@ -123,161 +106,42 @@ const PREPOSITIONS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"]; /// The Russian alphabet, when the dictionary has no `TRY` line. const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё"; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 3; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; +/// `е` written for `ё`, or the other way round, is barely a slip. const COST_YO: u32 = 3; -const COST_MAP: u32 = 7; -const COST_REP: u32 = 9; -const COST_SPLIT: u32 = 12; +/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`). const COST_SPLIT_PREPOSITION: u32 = 17; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} +/// What Russian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT }; -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The other members of each [`MAP`] group holding `c`. -fn alternatives(c: char) -> Vec { - let mut out: Vec = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect(); - out.sort_unstable(); - out.dedup(); - out -} - -/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip -/// (`ё` is optional in ordinary writing). +/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest. fn map_cost(from: char, to: char) -> u32 { - if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP } + if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP } } -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; +/// A missing space: cheap after a short leading word or before a short trailing one, dearer after +/// a short preposition or between two longer words, never leaving a stray short half. +fn split(a: &str, b: &str) -> Option { + let long = |p: &str| p.chars().count() >= 3; + // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for + // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. + let preposition = PREPOSITIONS.contains(&a); + if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { + return None; } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } + if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { + Some(COST_SPLIT_PREPOSITION) + } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None } } -/// A letter's group representative (the first of its first group), lower case: `ё` → `е`, -/// `з` → `з`, `с` → `з`. -fn fold(c: char) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l) -} - -/// Optimal string alignment distance where a slip within a letter group costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if fold(ai) == fold(bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} - /// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { suggest_in(&super::dictionary(), word, max) @@ -285,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Slips within a letter group. - let alts: Vec> = chars.iter().map(|&c| alternatives(c)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in REP { - for (i, _) in input.match_indices(from) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for - // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. - let preposition = PREPOSITIONS.contains(&a); - if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { - continue; - } - let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { - COST_SPLIT_PREPOSITION - } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = ALPHABET.chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a group relative of it, -/// either case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(fold).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -461,6 +161,20 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip + /// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip; + /// a preposition is never split off a verb, and after a one-letter one a typo is likelier. + #[test] + fn russian_costs_rank_its_slips() { + let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap(); + assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]); + assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]); + assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into())); + let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4); + assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}"); + assert!(s.contains(&"в доме".into()), "{s:?}"); + } + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the /// shared Latin-script check doesn't reach it. @@ -518,12 +232,15 @@ mod tests { #[test] fn distance_weighs_letter_groups_half() { + use crate::affix_suggest::{fold, weighted_distance}; + let map: Vec> = MAP.iter().map(|g| g.chars().collect()).collect(); + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); let c = |s: &str| s.chars().collect::>(); - assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1); - assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1); - assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2); - assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6); - assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3); + assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1); + assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1); + assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2); + assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6); + assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3); } #[test] From 9a8f4669fa9485853db9eadc668e05eb2f4dda2a Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:55:43 -0100 Subject: [PATCH 06/21] Proofing: one language table instead of nine lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib.rs named every dictionary language in nine places: reloading for a new folder, the installed list, load errors, the missing-dictionary hint, the loaded check, the spelling message, spelling, suggestions and grammar. The lists had drifted. dictionary_load_error had no Russian or Czech entry, so a Russian or Czech dictionary that failed to load showed no reason in File › Options › Proofing, though the status bar, which reads another list, said why. LANGUAGES holds one row per language: its dictionary slot, its spelling message, and its spelling, suggestion and grammar functions. The nine functions read the row, so a language now joins with its module, its ProofLang and one row. The per-language reload_dictionary and installed_path wrappers had no callers left, inside the crate or out, and are gone. Tests: every_dictionary_language_reports_its_load_error, which fails for Russian on the old code, and the_language_table_covers_the_proofed_languages (each ProofLang::PROOFED language but English has exactly one row). Proof 285, engine 927 and layout 501 (lib). --- crates/proof/src/cs/mod.rs | 14 +- crates/proof/src/es/mod.rs | 23 +--- crates/proof/src/hr/mod.rs | 14 +- crates/proof/src/id/mod.rs | 14 +- crates/proof/src/it/mod.rs | 14 +- crates/proof/src/lib.rs | 274 ++++++++++++++++++++++--------------- crates/proof/src/pl/mod.rs | 14 +- crates/proof/src/pt/mod.rs | 14 +- crates/proof/src/ru/mod.rs | 14 +- crates/proof/src/slot.rs | 31 +++++ crates/proof/src/tr/mod.rs | 13 +- 11 files changed, 208 insertions(+), 231 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 55e21aef6..1d72dc913 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -18,7 +18,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; /// The Czech dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); /// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off @@ -33,18 +33,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Czech dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 253642c83..e16e9d6bd 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -25,10 +25,10 @@ use crate::slot::DictSlot; pub use suggest::suggest; /// The dictionary of each variety, found on first use (see [`DictSlot`]). -static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); -static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); -static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); -static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); +pub(crate) static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +pub(crate) static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +pub(crate) static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +pub(crate) static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); /// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the /// generic one RLA-ES publishes). @@ -72,21 +72,6 @@ pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), S slot(lang).install(aff, dic) } -/// Forget every variety's loaded dictionary, so the next check looks for an installed one again -/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - for slot in [&SPAIN_DICT, &MEXICO_DICT, &LATIN_AMERICA_DICT, &URUGUAY_DICT] { - slot.reload(); - } -} - -/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder -/// or the system locations now (File › Options › Proofing shows it); `None` when there is none, -/// or when one given with [`install_dictionary`] is in use. -pub fn installed_path(lang: ProofLang) -> Option { - slot(lang).installed_path() -} - /// Is a dictionary available for `lang`'s variety (spelling is checked)? pub fn has_dictionary(lang: ProofLang) -> bool { slot(lang).has_dictionary() diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 6372a47d2..036b72350 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -20,7 +20,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; /// The Croatian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); /// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off @@ -35,18 +35,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Croatian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Croatian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 65b820dfc..f1e44df45 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -21,7 +21,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; /// The Indonesian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); /// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off @@ -36,18 +36,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Indonesian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is an Indonesian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 168883e17..1e606ad74 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -29,7 +29,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; /// The Italian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); /// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Italian spelling is off @@ -44,18 +44,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Italian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is an Italian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 4bf4da78b..0c51e2362 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -159,57 +159,150 @@ pub fn ignore_word(w: &str) { } } +/// A language proofed against an installed Hunspell dictionary: all the functions below need to +/// know about it, so a language joins with one row in [`LANGUAGES`]. +struct Language { + lang: ProofLang, + slot: &'static dyn slot::Slot, + /// The spelling mark's message, in the language. + message: &'static str, + is_correct: fn(&str, ProofLang, &Options) -> bool, + suggest: fn(&str, usize, ProofLang) -> Vec, + /// Grammar issues; the flag says whether the first word must have a capital (not in a list + /// item). + grammar: fn(&str, ProofLang, bool) -> Vec, +} + +/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing +/// lists them. English has a built-in word list and is handled apart. +static LANGUAGES: [Language; 12] = [ + Language { + lang: ProofLang::Pl, + slot: &pl::DICT, + message: "Możliwy błąd pisowni", + is_correct: |w, _, o| pl::is_correct_with_options(w, o), + suggest: |w, n, _| pl::suggest(w, n), + grammar: |t, _, c| pl::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Pt, + slot: &pt::DICT, + message: "Possível erro de ortografia", + is_correct: |w, _, o| pt::is_correct_with_options(w, o), + suggest: |w, n, _| pt::suggest(w, n), + grammar: |t, _, c| pt::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::It, + slot: &it::DICT, + message: "Possibile errore di ortografia", + is_correct: |w, _, o| it::is_correct_with_options(w, o), + suggest: |w, n, _| it::suggest(w, n), + grammar: |t, _, c| it::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Hr, + slot: &hr::DICT, + message: "Moguća pravopisna pogreška", + is_correct: |w, _, o| hr::is_correct_with_options(w, o), + suggest: |w, n, _| hr::suggest(w, n), + grammar: |t, _, c| hr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Tr, + slot: &tr::DICT, + message: "Olası yazım hatası", + is_correct: |w, _, o| tr::is_correct_with_options(w, o), + suggest: |w, n, _| tr::suggest(w, n), + grammar: |t, _, c| tr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Ru, + slot: &ru::DICT, + message: "Возможна орфографическая ошибка", + is_correct: |w, _, o| ru::is_correct_with_options(w, o), + suggest: |w, n, _| ru::suggest(w, n), + grammar: |t, _, c| ru::grammar::check_with(t, c), + }, + // Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no + // exception. + Language { + lang: ProofLang::Cs, + slot: &cs::DICT, + message: "Možná pravopisná chyba", + is_correct: |w, _, o| cs::is_correct_with_options(w, o), + suggest: |w, n, _| cs::suggest(w, n), + grammar: |t, _, _| cs::grammar::check(t), + }, + Language { + lang: ProofLang::Id, + slot: &id::DICT, + message: "Kemungkinan salah eja", + is_correct: |w, _, o| id::is_correct_with_options(w, o), + suggest: |w, n, _| id::suggest(w, n), + grammar: |t, _, _| id::grammar::check(t), + }, + Language { + lang: ProofLang::Es, + slot: &es::SPAIN_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsMx, + slot: &es::MEXICO_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::Es419, + slot: &es::LATIN_AMERICA_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsUy, + slot: &es::URUGUAY_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, +]; + +/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof. +fn language(lang: ProofLang) -> Option<&'static Language> { + LANGUAGES.iter().find(|l| l.lang == lang) +} + /// Search `dir` for Hunspell dictionaries before the system locations (File › Options › /// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from /// the new search path, and text is checked again. pub fn set_dictionaries_folder(dir: Option) { hunspell::set_dictionary_dir(dir); - pl::reload_dictionary(); - pt::reload_dictionary(); - it::reload_dictionary(); - hr::reload_dictionary(); - tr::reload_dictionary(); - ru::reload_dictionary(); - cs::reload_dictionary(); - id::reload_dictionary(); - es::reload_dictionary(); + for l in &LANGUAGES { + l.slot.reload(); + } DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } /// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it /// now (`None`: not installed, so the language has no spelling marks). pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> { - vec![ - (ProofLang::Pl, pl::installed_path()), - (ProofLang::Pt, pt::installed_path()), - (ProofLang::It, it::installed_path()), - (ProofLang::Hr, hr::installed_path()), - (ProofLang::Tr, tr::installed_path()), - (ProofLang::Ru, ru::installed_path()), - (ProofLang::Cs, cs::installed_path()), - (ProofLang::Id, id::installed_path()), - (ProofLang::Es, es::installed_path(ProofLang::Es)), - (ProofLang::EsMx, es::installed_path(ProofLang::EsMx)), - (ProofLang::Es419, es::installed_path(ProofLang::Es419)), - (ProofLang::EsUy, es::installed_path(ProofLang::EsUy)), - ] + LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect() } /// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse /// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has /// checked yet reports `None` (#2830). pub fn dictionary_load_error(lang: ProofLang) -> Option { - let key = match lang { - ProofLang::Pl => "pl_PL", - ProofLang::Pt => "pt_BR", - ProofLang::It => "it_IT", - ProofLang::Hr => "hr_HR", - ProofLang::Tr => "tr_TR", - ProofLang::Id => "id_ID", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang), - _ => return None, - }; - hunspell::load_error(key).map(|(_, reason)| reason) + hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason) } /// A number that changes whenever a word is added to or removed from the user dictionary, or the @@ -440,36 +533,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p /// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English). fn dictionary_loaded(lang: ProofLang) -> Option { - Some(match lang { - ProofLang::Pl => pl::has_dictionary(), - ProofLang::Pt => pt::has_dictionary(), - ProofLang::It => it::has_dictionary(), - ProofLang::Hr => hr::has_dictionary(), - ProofLang::Tr => tr::has_dictionary(), - ProofLang::Ru => ru::has_dictionary(), - ProofLang::Cs => cs::has_dictionary(), - ProofLang::Id => id::has_dictionary(), - l if l.is_spanish() => es::has_dictionary(l), - _ => return None, - }) + language(lang).map(|l| l.slot.has_dictionary()) } /// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for; /// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`). fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> { - match lang { - _ if installed => None, - ProofLang::Pl => Some("pl_PL"), - ProofLang::Pt => Some("pt_BR"), - ProofLang::It => Some("it_IT"), - ProofLang::Hr => Some("hr_HR"), - ProofLang::Tr => Some("tr_TR"), - ProofLang::Ru => Some("ru_RU"), - ProofLang::Cs => Some("cs_CZ"), - ProofLang::Id => Some("id_ID"), - l if l.is_spanish() => Some(es::dictionary_name(l)), - _ => None, - } + language(lang).filter(|_| !installed).map(|l| l.slot.name()) } /// Is `word` spelled correctly in the run's proofing language? A language without a dictionary @@ -739,18 +809,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { return None; } - let message = match lang { - ProofLang::Pl => "Możliwy błąd pisowni", - ProofLang::Hr => "Moguća pravopisna pogreška", - ProofLang::Pt => "Possível erro de ortografia", - ProofLang::It => "Possibile errore di ortografia", - ProofLang::Tr => "Olası yazım hatası", - ProofLang::Ru => "Возможна орфографическая ошибка", - ProofLang::Cs => "Možná pravopisná chyba", - ProofLang::Id => "Kemungkinan salah eja", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico", - _ => "Possible spelling mistake", - }; + let message = language(lang).map_or("Possible spelling mistake", |l| l.message); Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) } @@ -774,40 +833,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> { /// checked as its composed form ([`composed`]). pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts), - ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts), - ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts), - ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts), - ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts), - ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct_with_options(word, opts), - ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct_with_options(word, opts), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => { - accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts) - } - ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts), - ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts), - ProofLang::Bg | ProofLang::De | ProofLang::Other => true, + if lang.is_english() { + return is_correct_english(word, lang, opts); } + // A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks. + language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts)) } /// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed /// ([`composed`]) like the dictionaries' words. pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang), - ProofLang::Pl => pl::suggest(word, max), - ProofLang::Hr => hr::suggest(word, max), - ProofLang::Pt => pt::suggest(word, max), - ProofLang::It => it::suggest(word, max), - ProofLang::Tr => tr::suggest(word, max), - ProofLang::Ru => ru::suggest(word, max), - ProofLang::Cs => cs::suggest(word, max), - ProofLang::Id => id::suggest(word, max), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + if lang.is_english() { + return suggest_english(word, max, lang); } + language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang)) } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones @@ -824,18 +864,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), - ProofLang::Pl => pl::grammar::check_with(text, capital_first), - ProofLang::Hr => hr::grammar::check_with(text, capital_first), - ProofLang::Pt => pt::grammar::check_with(text, capital_first), - ProofLang::It => it::grammar::check_with(text, capital_first), - ProofLang::Tr => tr::grammar::check_with(text, capital_first), - ProofLang::Ru => ru::grammar::check_with(text, capital_first), - // Czech has no sentence-capital rule, so a list item needs no exception. - ProofLang::Cs => cs::grammar::check(text), - ProofLang::Id => id::grammar::check(text), - // Spanish has no sentence-capital rule either. - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + _ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)), } } @@ -1133,6 +1162,33 @@ mod tests { assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); } + /// A dictionary that fails to load says why in File › Options › Proofing in every language + /// that reads one; Russian and Czech were missing from the list that named their dictionaries + /// there. + #[test] + fn every_dictionary_language_reports_its_load_error() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for (lang, _) in installed_dictionaries() { + let name = dictionary_needed(lang, false).unwrap(); + let reason = format!("zq unreadable {name}"); + let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone())); + assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}"); + hunspell::clear_load_error(name); + } + } + + /// Every proofed language other than English has a row in the language table, and only those. + #[test] + fn the_language_table_covers_the_proofed_languages() { + for l in ProofLang::PROOFED { + assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}"); + } + for (i, row) in LANGUAGES.iter().enumerate() { + assert!(row.lang.is_proofed(), "{:?}", row.lang); + assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang); + } + } + /// #1741: a list item may be a fragment, so its first word needs no capital in any language /// with the rule; a plain paragraph's does, and later sentences in the item are still checked. #[test] diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 4d79986d9..30d6d966f 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -28,7 +28,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; /// The Polish dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); /// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Polish spelling is off @@ -43,18 +43,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Polish dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Polish dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index 860cf4579..0110a8c18 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -27,7 +27,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; /// The Portuguese dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); /// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off @@ -42,18 +42,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Portuguese dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Portuguese dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index e46bf305f..67be70858 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -39,7 +39,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; /// The Russian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); /// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Russian spelling is off @@ -54,18 +54,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Russian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs index 3c0f2bd35..8b90a94a9 100644 --- a/crates/proof/src/slot.rs +++ b/crates/proof/src/slot.rs @@ -30,6 +30,37 @@ impl Dictionary for AffixDict { } } +/// What every language's slot answers, whatever its dictionary type (the language table in +/// lib.rs holds them side by side). +pub(crate) trait Slot: Sync { + /// See [`DictSlot::name`]. + fn name(&self) -> &'static str; + /// See [`DictSlot::installed_path`]. + fn installed_path(&self) -> Option; + /// See [`DictSlot::has_dictionary`]. + fn has_dictionary(&self) -> bool; + /// See [`DictSlot::reload`]. + fn reload(&self); +} + +impl Slot for DictSlot { + fn name(&self) -> &'static str { + DictSlot::name(self) + } + + fn installed_path(&self) -> Option { + DictSlot::installed_path(self) + } + + fn has_dictionary(&self) -> bool { + DictSlot::has_dictionary(self) + } + + fn reload(&self) { + DictSlot::reload(self); + } +} + /// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on /// disk (it then has no file to report, even if a matching file also happens to exist): one lock, /// so a reload and an install can't interleave their two fields. diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index eb5f0ea52..f247336be 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -29,7 +29,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; /// The Turkish dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); /// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available. @@ -43,17 +43,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again. -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Turkish dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Turkish dictionary available? pub fn has_dictionary() -> bool { DICT.has_dictionary() From e0014ff1d9a2aed00ca1f464b79b642c01564c74 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:58:30 -0100 Subject: [PATCH 07/21] Proofing: the spelling options are read in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Czech, Croatian, Indonesian, Italian, Polish, Portuguese and Spanish began their spelling checks with the same thirteen lines, byte for byte: no dictionary means no marks, single letters and words without letters pass, words with digits and short words in capitals pass when the File › Options › Proofing choices say so, and words in another script pass. Each also had its own copy of the Latin-script test. These were the lines #3966 had to fix in six copies one by one, and where the Russian and Turkish copies stayed wrong until the first commit here. spelling.rs has them once (`word_to_check`), and each language goes on from there with its own rules (Polish apostrophe endings, Italian elisions, Indonesian clitics, Uruguayan `vos` forms). Tests: words_needing_no_lookup. Ignoring the uppercase option there fails it and all seven languages' proofing-options tests. Proof: 286 tests. --- crates/proof/src/cs/mod.rs | 21 ++------------- crates/proof/src/es/mod.rs | 21 ++------------- crates/proof/src/hr/mod.rs | 21 ++------------- crates/proof/src/id/mod.rs | 21 ++------------- crates/proof/src/it/mod.rs | 21 ++------------- crates/proof/src/lib.rs | 1 + crates/proof/src/pl/mod.rs | 21 ++------------- crates/proof/src/pt/mod.rs | 21 ++------------- crates/proof/src/spelling.rs | 52 ++++++++++++++++++++++++++++++++++++ 9 files changed, 67 insertions(+), 133 deletions(-) create mode 100644 crates/proof/src/spelling.rs diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 1d72dc913..9f8bbd0a7 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -38,11 +38,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals /// (acronyms: `ČR`, `EU`), words with digits and words in other scripts are accepted, and so /// is every word when no Czech dictionary is installed. @@ -64,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Czech dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index e16e9d6bd..5263bd1d9 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -77,11 +77,6 @@ pub fn has_dictionary(lang: ProofLang) -> bool { slot(lang).has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in /// capitals (acronyms: `ONU`, `CDMX`), words with digits and words in other scripts are accepted, /// and so is every word when no dictionary is installed for the variety. Hyphenated compounds @@ -105,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Spanish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang)); } diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 036b72350..53edf0674 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -40,11 +40,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals /// (acronyms: `HRT`, `RH`), words with digits and words in other scripts are accepted, and so /// is every word when no Croatian dictionary is installed. @@ -66,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Croatian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index f1e44df45..04248d638 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -45,11 +45,6 @@ pub fn has_dictionary() -> bool { /// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; -/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals /// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so /// is every word when no Indonesian dictionary is installed. A hyphenated word is correct when @@ -73,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Indonesian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w) || w.split('-').all(|p| { diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 1e606ad74..ee5aaaee3 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -49,11 +49,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the /// apostrophe is what gets checked (`l'amico`, `un'idea`, `c'è`, `m'hai`, `dov'è`, `anch'io`). const PROCLITICS: &[&str] = &[ @@ -93,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Italian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let lower = w.to_lowercase(); if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) { return true; diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 0c51e2362..365229d70 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -67,6 +67,7 @@ pub mod pl; pub mod pt; pub mod ru; mod slot; +mod spelling; pub mod tr; pub mod vi; diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 30d6d966f..7cde9d0f9 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -52,11 +52,6 @@ pub fn has_dictionary() -> bool { /// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`). const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"]; -/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals /// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so /// is every word when no Polish dictionary is installed. @@ -78,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Polish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let w = w.replace('’', "'"); if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index 0110a8c18..ea91dbdb1 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -47,11 +47,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals /// (acronyms: `USP`, `PT`), words with digits and words in other scripts are accepted, and so /// is every word when no Portuguese dictionary is installed. @@ -73,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Portuguese dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs new file mode 100644 index 000000000..ab5e520e9 --- /dev/null +++ b/crates/proof/src/spelling.rs @@ -0,0 +1,52 @@ +//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian, +//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the +//! File › Options › Proofing choices. Each language goes on from there with its own rules. + +use crate::Options; +use crate::affix::AffixDict; + +/// Is `c` a letter of the Latin script (the only one these dictionaries cover)? +fn latin(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it +/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single +/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the +/// options skip those, or it is in another script. +pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> { + if d.is_empty() { + return None; + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + return None; + } + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return None; + } + w.chars().any(latin).then_some(w) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn words_needing_no_lookup() { + let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); + let defaults = Options::default(); + assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary"); + for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] { + assert_eq!(word_to_check(&d, w, &defaults), None, "{w}"); + } + assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot")); + assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot")); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO")); + assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B")); + assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms"); + } +} From 4dfdecf3d716101519f186125d9f5ab29eb39d10 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 20:57:35 -0100 Subject: [PATCH 08/21] Proofing: a candidate found again at a lower cost ranks at that cost The suggestion rounds don't run in rising cost: a MAP diacritic drop (11) comes before a REP confusion (10, 9 for Russian). The engine checked each candidate once, at the first cost it was offered, so the same word found again more cheaply kept the dearer cost and could rank one place too low. Czech's engine always did this, and the merged engine took it over. The candidate set is now a map to the cheapest cost offered so far, and an offer that is no cheaper is skipped as before. Found by CodeRabbit on #4191. Tests: a_cheaper_later_offer_wins; skipping every repeat offer again fails it. Proof: 288 tests. --- crates/proof/src/affix_suggest.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 1da355595..ade16edd1 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -102,7 +102,8 @@ struct Collector<'a> { d: &'a AffixDict, /// The word, lower case. input: String, - tested: HashSet, + /// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost). + tested: HashMap, /// Suggestion → (cost, order found). found: HashMap, seq: usize, @@ -112,9 +113,16 @@ impl Collector<'_> { /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every /// word is in the dictionary, as written or capitalized (names). fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + if cand.is_empty() || cand == self.input { return; } + match self.tested.get_mut(cand) { + Some(prev) if *prev <= cost => return, + Some(prev) => *prev = cost, + None => { + self.tested.insert(cand.to_string(), cost); + } + } let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { @@ -270,7 +278,7 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let case = casing(word); let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; // 1. Letter case. if d.contains(&input) && word != input { col.keep(input.clone(), COST_CASE); @@ -463,6 +471,18 @@ mod tests { assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); } + /// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop) + /// ranks at the lower one. + #[test] + fn a_cheaper_later_offer_wins() { + let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); + let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + col.offer("cafe", COST_DROP_DIACRITIC); + col.offer("cafe", COST_REP); + col.offer("cafe", COST_SCAN); + assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP)); + } + #[test] fn distance_weighs_map_slips_half() { let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; From 4c9ff5db3584b2052a2940bf45da6e52f06b7578 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 20:57:37 -0100 Subject: [PATCH 09/21] Proofing: a dictionary load from before a folder change is not kept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dictionary is read outside the lock. If the dictionaries folder changed while it was being read (File › Options › Proofing), the reload emptied the slot first and the late load then filled it with the old folder's dictionary. It stayed until the next reload. Every language's copy of the loader had this window; the shared slot now closes it in one place. The slot counts reloads. A load that started before a reload still answers the check that asked for it, but it isn't kept: the next check reads the new folder. Found by CodeRabbit on #4191. Tests: a_load_from_before_a_reload_is_not_kept; not counting reloads fails it. Proof: 288 tests. --- crates/proof/src/slot.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs index 8b90a94a9..6255ae5a5 100644 --- a/crates/proof/src/slot.rs +++ b/crates/proof/src/slot.rs @@ -67,6 +67,8 @@ impl Slot for DictSlot { struct State { dict: Option>, in_memory: bool, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, } /// One language's dictionary. @@ -80,7 +82,7 @@ pub(crate) struct DictSlot { impl DictSlot { pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { - DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false }) } + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) } } fn read(&self) -> RwLockReadGuard<'_, State> { @@ -105,12 +107,25 @@ impl DictSlot { /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or /// an empty one when none is available (the language's spelling is off then). pub(crate) fn get(&self) -> Arc { - if let Some(d) = self.read().dict.as_ref() { - return d.clone(); + let reloads = { + let s = self.read(); + if let Some(d) = s.dict.as_ref() { + return d.clone(); + } + s.reloads + }; + self.keep(reloads, Arc::new(self.load_installed())) + } + + /// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the + /// load may have read the old folder: it serves this one check and the next one loads again. + fn keep(&self, reloads: u64, loaded: Arc) -> Arc { + let mut s = self.write(); + if s.reloads != reloads { + return s.dict.clone().unwrap_or(loaded); } - let loaded = Arc::new(self.load_installed()); // Another thread (or an install) may have filled it meanwhile: that one wins. - self.write().dict.get_or_insert(loaded).clone() + s.dict.get_or_insert(loaded).clone() } fn load_installed(&self) -> D { @@ -142,6 +157,7 @@ impl DictSlot { let mut slot = self.write(); slot.dict = None; slot.in_memory = false; + slot.reloads = slot.reloads.wrapping_add(1); } /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the @@ -178,4 +194,17 @@ mod tests { assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); assert_eq!(SLOT.name(), "zz_SLOT_TEST"); } + + /// A load that started before a dictionaries-folder change is not kept after its reload. + #[test] + fn a_load_from_before_a_reload_is_not_kept() { + let slot: DictSlot = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test"); + let before = slot.read().reloads; + slot.reload(); + let _ = slot.keep(before, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_none(), "the next check loads from the new folder"); + let now = slot.read().reloads; + let _ = slot.keep(now, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_some(), "a load since the reload is kept"); + } } From 46a3067e2077a100358f32e614374f0c088eb587 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:59:12 +0200 Subject: [PATCH 10/21] proof: flag aliases, single-byte flags, ICONV, IGNORE and KEEPCASE in affix dictionaries (#735) The Hungarian Hunspell dictionary declares SET UTF-8 but writes its flags as single bytes, uses AF aliases and 158 distinct flags. Read flag fields byte by byte unless FLAG UTF-8 is set, expand AF aliases, skip AM, apply ICONV and IGNORE, honour KEEPCASE, and widen flag masks to 256 bits. Compounding and NEEDAFFIX stay unsupported for now. --- crates/proof/src/affix.rs | 454 ++++++++++++++++++++++++++--------- crates/proof/src/hunspell.rs | 186 +++++++++++++- 2 files changed, 530 insertions(+), 110 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 4ec068466..4ce915afa 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -11,10 +11,14 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //! -//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), -//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross -//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), -//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding, +//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`), +//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and +//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes +//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the +//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` +//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` +//! and `REP`. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding, //! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and @@ -33,8 +37,6 @@ const EMPTY: u32 = u32::MAX; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ - "AF", - "AM", "CHECKSHARPS", "CIRCUMFIX", "COMPLEXPREFIXES", @@ -46,17 +48,78 @@ const UNSUPPORTED: &[&str] = &[ "COMPOUNDPERMITFLAG", "COMPOUNDRULE", "FULLSTRIP", - "ICONV", - "IGNORE", - "KEEPCASE", - "LEMMA_PRESENT", "NEEDAFFIX", "OCONV", "ONLYINCOMPOUND", "PSEUDOROOT", - "SUBSTANDARD", ]; +/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary +/// uses 158). +const MAX_FLAGS: usize = 256; + +/// A set of flags, one bit per distinct flag character of the dictionary. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +struct Mask([u64; MAX_FLAGS / 64]); + +impl Mask { + const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]); + + /// The set holding flag number `i` alone (empty when `i` is out of range). + fn bit(i: usize) -> Mask { + let mut m = Mask::EMPTY; + if let Some(w) = m.0.get_mut(i / 64) { + *w = 1 << (i % 64); + } + m + } + + /// The number of the lowest flag in the set, if any. + fn first(self) -> Option { + self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize) + } + + fn is_empty(self) -> bool { + self.0.iter().all(|&w| w == 0) + } + + /// Do the two sets share a flag? + fn meets(self, other: Mask) -> bool { + !(self & other).is_empty() + } + + /// Does this set hold every flag of `other`? + fn covers(self, other: Mask) -> bool { + (self & other) == other + } +} + +impl std::ops::BitAnd for Mask { + type Output = Mask; + fn bitand(mut self, o: Mask) -> Mask { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a &= b; + } + self + } +} + +impl std::ops::BitOr for Mask { + type Output = Mask; + fn bitor(mut self, o: Mask) -> Mask { + self |= o; + self + } +} + +impl std::ops::BitOrAssign for Mask { + fn bitor_assign(&mut self, o: Mask) { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a |= b; + } + } +} + /// One character class of a rule's condition. #[derive(Clone, Debug, PartialEq, Eq)] enum Cond { @@ -130,14 +193,14 @@ const MAX_FORMS_WORK: usize = 1 << 18; #[derive(Clone, Debug)] struct Rule { /// Bit of the rule's flag in the stems' flag masks. - bit: u128, + bit: Mask, flag: char, cross: bool, strip: Box, add: Box, /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. - cont: u128, + cont: Mask, cond: Box<[Cond]>, cond_text: Box, } @@ -174,11 +237,11 @@ pub struct AffixDict { /// Per stem: index into `sets`. flags: Vec, /// Distinct flag masks. - sets: Vec, + sets: Vec, /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). - forbidden: u128, + forbidden: Mask, /// Stems with NOSUGGEST are correct, but none of their forms are suggested. - no_suggest: u128, + no_suggest: Mask, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, /// Flag char of each bit. @@ -195,6 +258,12 @@ pub struct AffixDict { key: String, map: Vec>, rep: Vec<(String, String)>, + /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none). + keepcase: Mask, + /// `ICONV`: input conversions applied before a lookup, `(from, to)`. + iconv: Vec<(String, String)>, + /// `IGNORE`: characters removed from input words (and from the stems and affixes). + ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, } @@ -263,6 +332,49 @@ struct Rules { /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option, no_suggest: Option, + /// The `KEEPCASE` flag. + keepcase: Option, + iconv: Vec<(String, String)>, + ignore: Vec, + /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. + aliases: Vec, +} + +impl Rules { + /// Flags as written in a `.dic` entry or a continuation class: an `AF` alias number when the + /// dictionary declares aliases, otherwise the flag characters themselves. + fn flags<'a>(&'a self, written: &'a str) -> Result<&'a str, String> { + if self.aliases.is_empty() || written.is_empty() || !written.bytes().all(|b| b.is_ascii_digit()) { + return Ok(written); + } + written + .parse::() + .ok() + .and_then(|n| n.checked_sub(1)) + .and_then(|i| self.aliases.get(i)) + .map(String::as_str) + .ok_or_else(|| format!("flag alias {written} is not defined")) + } + + /// `w` without the `IGNORE` characters. + fn strip_ignored(&self, w: &str) -> String { + if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } + } +} + +/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a +/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag. +fn split_fields(line: &str) -> impl Iterator { + line.split([' ', '\t']).filter(|f| !f.is_empty()) +} + +/// The one flag character a directive (`FORBIDDENWORD w`) names. +fn one_flag(arg: &str, dir: &str, n: usize) -> Result { + let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); + match (it.next(), it.next()) { + (Some(c), None) if !c.is_whitespace() => Ok(c), + _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), + } } /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. @@ -304,8 +416,10 @@ fn parse_aff(aff: &str) -> Result { let mut cross: HashMap<(bool, char), bool> = HashMap::new(); let mut map_header = false; let mut rep_header = false; + let mut af_header = false; + let mut iconv_header = false; for (n, line) in aff.lines().enumerate() { - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let Some(&dir) = f.first() else { continue }; if dir.starts_with('#') { continue; @@ -314,25 +428,33 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + // Morphological aliases, and flags that only matter for morphological generation + // and suggestions: the words they mark are correct. + "AM" | "SUBSTANDARD" | "LEMMA_PRESENT" => {} + "AF" => { + if !af_header && f.len() == 2 && at(1).parse::().is_ok() { + af_header = true; + } else { + r.aliases.push(at(1).to_string()); + } + } + "ICONV" => { + if !iconv_header && f.len() == 2 && at(1).parse::().is_ok() { + iconv_header = true; + } else if f.len() >= 3 { + r.iconv.push((at(1).to_string(), at(2).to_string())); + } + } + "IGNORE" => r.ignore = at(1).chars().collect(), + "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work // with the one-character parsing here; only the multi-character encodings differ. "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } "FLAG" => {} - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - r.forbidden = Some(c); - } else { - r.no_suggest = Some(c); - } - } - _ => return Err(format!("line {}: {directive} needs one flag character", n + 1)), - } - } + "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?), + "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), "TRY" => r.try_chars = cap_try(at(1)), "KEY" => r.key = at(1).to_string(), "MAP" => { @@ -383,6 +505,19 @@ fn parse_aff(aff: &str) -> Result { _ => {} } } + // Aliases may be declared after the rules that use them; resolve once all are read. + let resolve = |rules: &mut Vec, r: &Rules| -> Result<(), String> { + for rule in rules.iter_mut() { + rule.4 = r.flags(&rule.4)?.to_string(); + rule.2 = r.strip_ignored(&rule.2); + rule.3 = r.strip_ignored(&rule.3); + } + Ok(()) + }; + let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); + resolve(&mut pfx, &r)?; + resolve(&mut sfx, &r)?; + (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -397,7 +532,7 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { - let Some(tok) = line.split_whitespace().next() else { continue }; + let Some(tok) = split_fields(line).next() else { continue }; if n == 0 && tok.parse::().is_ok() { continue; } @@ -420,9 +555,11 @@ impl AffixDict { word.push(c); } } + let word = rules.strip_ignored(&word); if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } + let flags = rules.flags(&flags).map_err(|e| format!(".dic line {}: {e}", n + 1))?.to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -434,11 +571,12 @@ impl AffixDict { flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); flag_chars.sort_unstable(); flag_chars.dedup(); - if flag_chars.len() > 128 { - return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len())); + if flag_chars.len() > MAX_FLAGS { + return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); } - let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i); - let forbidden = rules.forbidden.map_or(0, bit_of); + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit); + let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of); + let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); // Stems sorted by bytes; homonyms (the same word twice) are merged. stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); let mut d = AffixDict { @@ -446,7 +584,10 @@ impl AffixDict { offs: Vec::with_capacity(stems.len() + 1), flag_chars: flag_chars.clone(), forbidden, - no_suggest: rules.no_suggest.map_or(0, bit_of), + no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), + keepcase, + iconv: rules.iconv, + ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, key: rules.key, map: rules.map, @@ -455,16 +596,16 @@ impl AffixDict { ..AffixDict::default() }; d.offs.push(0); - let mut set_ids: HashMap = HashMap::new(); + let mut set_ids: HashMap = HashMap::new(); let mut i = 0; while i < stems.len() { let Some((word, _)) = stems.get(i) else { break }; - let mut mask = 0u128; + let mut mask = Mask::EMPTY; let mut j = i; while let Some((w, f)) = stems.get(j) && w == word { - mask |= f.chars().map(bit_of).fold(0, |a, b| a | b); + mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); j += 1; } let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; @@ -478,7 +619,7 @@ impl AffixDict { i = j; } let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { - let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c)); + let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) }; d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; @@ -493,7 +634,7 @@ impl AffixDict { // Each (appended, stripped) pair's group, found by hash rather than by scanning the // appended string's groups: a dictionary with many strips per suffix stays linear. let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); - let mut by_flag: Vec> = vec![Vec::new(); 128]; + let mut by_flag: Vec> = vec![Vec::new(); MAX_FLAGS]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; @@ -504,9 +645,7 @@ impl AffixDict { groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); } } - if r.bit != 0 - && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) - { + if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { list.push(i); } } @@ -551,12 +690,12 @@ impl AffixDict { } /// The flag characters of `mask`, in flag order (how the compact format stores flags). - fn flag_chars_for(&self, mask: u128) -> String { - self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect() + fn flag_chars_for(&self, mask: Mask) -> String { + self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() } /// The character of `mask` when it holds exactly one flag. - fn flag_char(&self, mask: u128) -> Option { + fn flag_char(&self, mask: Mask) -> Option { let s = self.flag_chars_for(mask); let mut it = s.chars(); match (it.next(), it.next()) { @@ -589,11 +728,12 @@ impl AffixDict { } /// Flag mask of a stem, if `w` is one. - fn stem_flags(&self, w: &str) -> Option { - self.stem_flags_excluding(w, self.forbidden) + fn stem_flags(&self, w: &str) -> Option { + self.stem_flags_except(w, Mask::EMPTY) } - fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option { + /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`. + fn stem_flags_except(&self, w: &str, deny: Mask) -> Option { if self.table.is_empty() { return None; } @@ -607,8 +747,9 @@ impl AffixDict { if self.word(t as usize) == Some(w) { let id = *self.flags.get(t as usize)?; let mask = self.sets.get(usize::from(id)).copied(); - // All derivations use this lookup: exclusions apply to stems and their forms. - return mask.filter(|m| m & excluded == 0); + // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, + // and (this is the only stem lookup) no affixed form is built on it either. + return mask.filter(|m| !m.meets(self.forbidden | deny)); } slot = (slot + 1) & mask; } @@ -621,8 +762,8 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one mask of their flags and /// followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { + fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool { + if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { return true; } if depth == 0 { @@ -641,9 +782,9 @@ impl AffixDict { .rules .iter() .filter_map(|&i| self.suffixes.get(i as usize)) - .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) - .fold(0u128, |m, r| m | r.bit); - if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true)) + .fold(Mask::EMPTY, |m, r| m | r.bit); + if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { return true; } } @@ -651,21 +792,50 @@ impl AffixDict { false } - /// Is `w` exactly (case and all) a stem or an affixed form of one? + /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE` + /// apply first. pub fn contains(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden) + self.contains_except(&self.normalize(w), Mask::EMPTY) } /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden | self.no_suggest) + self.contains_except(&self.normalize(w), self.no_suggest) } - fn contains_excluding(&self, w: &str, excluded: u128) -> bool { + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without + /// the `IGNORE` characters. + fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { + if self.iconv.is_empty() && self.ignore.is_empty() { + return std::borrow::Cow::Borrowed(w); + } + let mut out = String::with_capacity(w.len()); + let mut rest = w; + while let Some(c) = rest.chars().next() { + let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); + match hit { + Some((from, to)) => { + out.extend(to.chars().filter(|c| !self.ignore.contains(c))); + rest = rest.get(from.len()..).unwrap_or_default(); + } + None => { + if !self.ignore.contains(&c) { + out.push(c); + } + rest = rest.get(c.len_utf8()..).unwrap_or_default(); + } + } + } + std::borrow::Cow::Owned(out) + } + + /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't + /// count (case variants of `KEEPCASE` stems). + fn contains_except(&self, w: &str, deny: Mask) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) { + if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { return true; } let mut root = String::with_capacity(w.len() + 8); @@ -680,13 +850,13 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) { + if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { return true; } - if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) { + if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { return true; } - if r.cross && self.suffixed(&root, r.bit, excluded) { + if r.cross && self.suffixed(&root, r.bit, deny) { return true; } } @@ -695,7 +865,7 @@ impl AffixDict { /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must /// carry those flags too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool { + fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { let mut root = String::with_capacity(w.len() + 8); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. @@ -710,18 +880,18 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_excluding(&root, excluded) - && f & need == need - && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)) + if let Some(f) = self.stem_flags_except(&root, deny) + && f.covers(need) + && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. - if need == 0 { - let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); - if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + if need.is_empty() { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit); + if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { return true; } } @@ -732,26 +902,29 @@ impl AffixDict { /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals /// — in lower case (and capitalized), since a sentence may start with any word and headings - /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a + /// `KEEPCASE` stem matches only as written. pub fn check(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden) + self.check_except(w, Mask::EMPTY) } /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden | self.no_suggest) + self.check_except(w, self.no_suggest) } - fn check_excluding(&self, w: &str, excluded: u128) -> bool { - if self.contains_excluding(w, excluded) { + fn check_except(&self, w: &str, deny: Mask) -> bool { + let w = self.normalize(w); + if self.contains_except(&w, deny) { return true; } - match casing(w) { + let deny = deny | self.keepcase; + match casing(&w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded), + Casing::Initial => self.contains_except(&w.to_lowercase(), deny), Casing::Upper => { let lower = w.to_lowercase(); - self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded) + self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) } } } @@ -766,11 +939,11 @@ impl AffixDict { let mut cross_sfx = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once. - let mut chain: Vec<(String, u128)> = Vec::new(); - let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut chain: Vec<(String, Mask)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if f & r.bit == 0 || !r.cond_matches(stem, true) { + if !f.meets(r.bit) || !r.cond_matches(stem, true) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -778,7 +951,7 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 && queued.insert((form.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); @@ -804,7 +977,7 @@ impl AffixDict { } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 && queued.insert((next.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); @@ -815,7 +988,7 @@ impl AffixDict { } } for r in &self.prefixes { - if f & r.bit == 0 || !r.cond_matches(stem, false) { + if !f.meets(r.bit) || !r.cond_matches(stem, false) { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; @@ -839,11 +1012,11 @@ impl AffixDict { } /// The suffix rules carrying any flag of `mask`, in rule order. - fn suffixes_with(&self, mask: u128) -> Vec { + fn suffixes_with(&self, mask: Mask) -> Vec { let mut rules: Vec = Vec::new(); let mut lists = 0; for (bit, list) in self.sfx_by_flag.iter().enumerate() { - if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + if mask.meets(Mask::bit(bit)) && !list.is_empty() { rules.extend_from_slice(list); lists += 1; } @@ -904,13 +1077,19 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] { - if mask != 0 + for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("KEEPCASE", self.keepcase)] { + if !mask.is_empty() && let Some(c) = self.flag_char(mask) { out.push_str(&format!("{directive} {c}\n")); } } + for (a, b) in &self.iconv { + out.push_str(&format!("ICONV {a} {b}\n")); + } + if !self.ignore.is_empty() { + out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); + } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules { @@ -934,8 +1113,8 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0); - if mask != 0 { + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default(); + if !mask.is_empty() { raw.push(b'/'); raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); } @@ -962,26 +1141,23 @@ impl AffixDict { comments.push(c.to_string()); continue; } - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - "MAP" => rules.map.push(at(1).chars().collect()), - "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars(); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - rules.forbidden = Some(c); - } else { - rules.no_suggest = Some(c); - } - } - _ => return Err(format!("affix dictionary: bad {directive} in `{line}`")), + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "KEEPCASE") => { + let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; + match directive { + "FORBIDDENWORD" => rules.forbidden = Some(c), + "NOSUGGEST" => rules.no_suggest = Some(c), + _ => rules.keepcase = Some(c), } } + "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), + "IGNORE" => rules.ignore = at(1).chars().collect(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { let mut fc = at(1).chars(); let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; @@ -1223,7 +1399,7 @@ pub(crate) mod tests { #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin - // letter, the digits and eight accented letters: 70 flags, 64 < n <= 128. + // letter, the digits and eight accented letters: 70 flags, more than 64. let mut flags = String::new(); let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); let chars: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); @@ -1239,6 +1415,24 @@ pub(crate) mod tests { assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); } + #[test] + fn more_than_128_distinct_flags_load() { + // The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up. + let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect(); + let mut aff = String::from("SET UTF-8\n"); + for (i, c) in flags.chars().enumerate() { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n")); + } + let half: String = flags.chars().step_by(2).collect(); + let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap(); + assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130")); + assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("szóx198") && !back.contains("szóx199")); + let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); + assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err()); + } + #[test] fn forbidden_words_are_rejected() { // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying @@ -1364,6 +1558,52 @@ pub(crate) mod tests { assert!(d.contains("kot") && d.contains("koty")); } + #[test] + fn flag_aliases_stand_for_flag_sets() { + // `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags. + // Morphological aliases (`AM`) and the fields after a stem are skipped. + let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n"; + let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); + assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); + assert!(!d.contains("házban"), "the stem has a and b, not c"); + assert!(AffixDict::from_hunspell(aff, "1\nház/3\n", &[]).is_err(), "alias 3 is not defined"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); + } + + #[test] + fn iconv_and_ignore_normalize_input() { + // Ligatures read as their letters; `IGNORE` characters vanish from input and stems. + let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap(); + assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine")); + assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)")); + assert!(!d.check("finer")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); + } + + #[test] + fn keepcase_stems_match_only_as_written() { + let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap(); + assert!(d.check("pH") && d.check("km") && d.check("kms")); + assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms")); + assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("km") && !back.check("Km")); + } + + #[test] + fn fields_split_on_spaces_and_tabs_only() { + // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator. + let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n"; + let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap(); + assert!(d.contains("koty") && d.contains("kotem")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("koty") && back.contains("kotem")); + } + #[test] fn unsupported_and_broken_input_is_an_error() { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 3449c8cab..d9937a14c 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -218,13 +218,27 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the /// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R /// and Windows-1251 the ones older Russian dictionaries use). +/// +/// Under a UTF-8 `SET` without `FLAG UTF-8`, flags are still single bytes, as in Hunspell: the +/// Hungarian dictionary writes its affix flags and flag aliases as bytes above 0x7F that aren't +/// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per +/// byte); the words, affixes and conditions around them are UTF-8. pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); - let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); + let directive = |name: &str| { + head.lines().find_map(|l| { + let l = l.trim_start_matches('\u{feff}'); + l.strip_prefix(name).filter(|rest| rest.starts_with([' ', '\t'])).map(|rest| rest.trim().to_ascii_uppercase()) + }) + }; + let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); - Ok((text(aff, ".aff")?, text(dic, ".dic")?)) + if matches!(directive("FLAG").as_deref(), Some("UTF-8" | "UTF8")) { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); + } + Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?)) } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), @@ -235,6 +249,153 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// Directives whose one argument is a flag. +const FLAG_DIRECTIVES: &[&str] = &[ + "CIRCUMFIX", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDFORBIDFLAG", + "COMPOUNDLAST", + "COMPOUNDFIRST", + "COMPOUNDMIDDLE", + "COMPOUNDPERMITFLAG", + "COMPOUNDROOT", + "FORBIDDENWORD", + "FORCEUCASE", + "GENERATE", + "HU_KOTOHANGZO", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "NOSUGGEST", + "ONLYINCOMPOUND", + "ONLYROOT", + "PSEUDOROOT", + "SUBSTANDARD", + "WARN", +]; + +/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian +/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. +const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; + +/// One byte per char: a flag field read as single-byte flags. +fn bytes_as_flags(b: &[u8]) -> String { + b.iter().map(|&x| char::from(x)).collect() +} + +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). +fn text_and_flags(field: &[u8], what: &str) -> Result { + let mut slash = None; + let mut i = 0; + while let Some(&b) = field.get(i) { + if b == b'\\' && field.get(i + 1) == Some(&b'/') { + i += 2; + continue; + } + if b == b'/' { + slash = Some(i); + break; + } + i += 1; + } + let (text, flags) = match slash { + Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), + None => (field, &[][..]), + }; + let mut out = std::str::from_utf8(text).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(); + out.push_str(&bytes_as_flags(flags)); + Ok(out) +} + +/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from +/// its parts is the original). +fn fields(line: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0; + let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t')); + for (i, b) in line.iter().enumerate() { + let sep = matches!(b, b' ' | b'\t'); + if sep != in_sep { + out.push(line.get(start..i).unwrap_or_default()); + start = i; + in_sep = sep; + } + } + out.push(line.get(start..).unwrap_or_default()); + out +} + +/// A UTF-8 `.aff` with single-byte flags (see [`decode`]). +fn decode_utf8_aff(aff: &[u8]) -> Result { + let mut out = String::with_capacity(aff.len()); + for (n, line) in aff.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.is_ascii() + { + out.push_str(s); + continue; + } + let parts = fields(line); + // Field index among the non-separator parts. + let words: Vec = + parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect(); + let dir = words + .first() + .and_then(|&i| parts.get(i)) + .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) + .unwrap_or_default(); + let what = format!(".aff line {}", n + 1); + if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { + out.push_str(&String::from_utf8_lossy(line)); + continue; + } + // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or + // a flag; from there on the line is kept as text. + let comment_at = words.iter().skip(1).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + 1); + for (i, p) in parts.iter().enumerate() { + let field = words.iter().position(|&w| w == i); + let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); + let decoded = match (dir.as_str(), field) { + _ if in_comment => String::from_utf8_lossy(p).into_owned(), + ("AF" | "PFX" | "SFX" | "COMPOUNDRULE", Some(1)) => bytes_as_flags(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?, + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p), + _ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(), + }; + out.push_str(&decoded); + } + } + Ok(out) +} + +/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes. +fn decode_utf8_dic(dic: &[u8]) -> Result { + let mut out = String::with_capacity(dic.len()); + for (n, line) in dic.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii()) + { + out.push_str(s); + continue; + } + let what = format!(".dic line {}", n + 1); + // The first field is `word/flags`; morphological fields after it are UTF-8 text. + let end = line.iter().position(|b| matches!(b, b' ' | b'\t')).unwrap_or(line.len()); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?); + let rest = line.get(end..).unwrap_or_default(); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -280,6 +441,25 @@ mod tests { assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok()); } + #[test] + fn utf8_dictionaries_keep_single_byte_flags() { + // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, + // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 1\nAF \xd5a # 1\nSFX \xd5 Y 1\nSFX \xd5 0 ok/\xe9 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); + let (a, d) = decode(aff, dic).unwrap(); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("SFX \u{d5} 0 ok/\u{e9} . 7")); + assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); + assert!(!dict.check("házban")); + // `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8. + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8"); + assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8"); + } + #[test] fn decodes_russian_single_byte_encodings() { // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). From 7ebfcf0ee2e8153a8e9fca67f9e4bb98901ac208 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:08:15 +0200 Subject: [PATCH 11/21] proof: .aff comments start after a directive's fields, .dic entries end where Hunspell ends them --- crates/proof/src/hunspell.rs | 79 ++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index d9937a14c..608e94777 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -355,8 +355,11 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { continue; } // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or - // a flag; from there on the line is kept as text. - let comment_at = words.iter().skip(1).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + 1); + // a flag; from there on the line is kept as text. It can only start after the directive's + // own fields, since a field such as an affix condition may itself begin with `#`. + let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); + let first_free = comment_field(&dir, field_text); + let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); for (i, p) in parts.iter().enumerate() { let field = words.iter().position(|&w| w == i); let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); @@ -373,6 +376,43 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { Ok(out) } +/// The first field of an `.aff` line where a trailing `# comment` may start: after the +/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its +/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field. +fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { + let number = |k: usize| field(k).is_some_and(|f| f.parse::().is_ok()); + match dir { + "PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4, + "PFX" | "SFX" => 5, + "REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN" + if number(1) && field(2).is_none_or(|f| f.starts_with('#')) => + { + 2 + } + "REP" | "ICONV" | "OCONV" | "PHONE" => 3, + "CHECKCOMPOUNDPATTERN" => 4, + _ => 2, + } +} + +/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the +/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the first space. +pub(crate) fn dic_entry_end(line: &[u8]) -> usize { + let blank = |b: &u8| matches!(b, b' ' | b'\t'); + let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); + let tab = line.iter().position(|&b| b == b'\t'); + let end = match (morph, tab) { + (Some(m), Some(t)) => m.min(t), + (m, t) => match m.or(t) { + Some(e) => e, + None => line.iter().position(|&b| b == b' ').unwrap_or(line.len()), + }, + }; + // Spaces before the morphological field aren't part of the entry. + line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) +} + /// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes. fn decode_utf8_dic(dic: &[u8]) -> Result { let mut out = String::with_capacity(dic.len()); @@ -387,8 +427,9 @@ fn decode_utf8_dic(dic: &[u8]) -> Result { continue; } let what = format!(".dic line {}", n + 1); - // The first field is `word/flags`; morphological fields after it are UTF-8 text. - let end = line.iter().position(|b| matches!(b, b' ' | b'\t')).unwrap_or(line.len()); + // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); + // morphological fields after it are UTF-8 text. + let end = dic_entry_end(line); out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?); let rest = line.get(end..).unwrap_or_default(); out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?); @@ -471,6 +512,36 @@ mod tests { assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); } + #[test] + fn comments_start_after_the_directive_fields() { + let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string); + assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4); + assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`"); + assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2); + assert_eq!(comment_field("AF", f("AF Aa # 1")), 2); + assert_eq!(comment_field("REP", f("REP 12")), 2); + assert_eq!(comment_field("REP", f("REP # x")), 3); + assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4); + assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2); + // A byte flag after an append text that begins with `#` stays a flag, not a comment. + let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap(); + assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}"); + } + + #[test] + fn dic_entries_end_where_hunspell_ends_them() { + let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string(); + assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); + assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); + assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); + assert_eq!(end("word/A 123"), "word/A", "neither: the first space"); + assert_eq!(end("word"), "word"); + assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); + // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + } + #[test] fn libreoffice_extension_dirs_follow_the_language() { // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese From efa570a9bee6253f2b96583b11cba1e14163658b Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:53:01 +0200 Subject: [PATCH 12/21] proof: a one-flag field keeps its first byte, a directive reads only its first argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without FLAG UTF-8, Hunspell reads a field that names one flag (PFX/SFX, KEEPCASE, FORBIDDENWORD and the like) through decode_flag, which takes its first byte: SFX é is the flag 0xC3. Such a field no longer fails the load. SYLLABLENUM lists flags and is read byte by byte like AF. SET and FLAG read only their first argument, so FLAG UTF-8 with a trailing comment keeps Unicode flags. --- crates/proof/src/hunspell.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 608e94777..934520beb 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -228,7 +228,11 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let directive = |name: &str| { head.lines().find_map(|l| { let l = l.trim_start_matches('\u{feff}'); - l.strip_prefix(name).filter(|rest| rest.starts_with([' ', '\t'])).map(|rest| rest.trim().to_ascii_uppercase()) + // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8. + l.strip_prefix(name) + .filter(|rest| rest.starts_with([' ', '\t'])) + .and_then(|rest| rest.split([' ', '\t']).find(|f| !f.is_empty())) + .map(str::to_ascii_uppercase) }) }; let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); @@ -365,9 +369,12 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); let decoded = match (dir.as_str(), field) { _ if in_comment => String::from_utf8_lossy(p).into_owned(), - ("AF" | "PFX" | "SFX" | "COMPOUNDRULE", Some(1)) => bytes_as_flags(p), + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => bytes_as_flags(p), ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?, - (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p), + // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` + // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. + ("PFX" | "SFX", Some(1)) => bytes_as_flags(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p.get(..1).unwrap_or_default()), _ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(), }; out.push_str(&decoded); @@ -501,6 +508,28 @@ mod tests { assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8"); } + #[test] + fn single_flag_fields_keep_their_first_byte() { + // Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the + // flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9. + let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes(); + let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap(); + assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}"); + assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}"); + assert!(d.contains("kert/\u{c3}\u{a9}")); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("kert") && dict.check("kertok")); + assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); + } + + #[test] + fn a_comment_after_flag_utf8_keeps_unicode_flags() { + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument"); + } + #[test] fn decodes_russian_single_byte_encodings() { // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). From 9b91b64d60365f2706c468d116dbb9a068a20682 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:47:25 +0200 Subject: [PATCH 13/21] proof: test that unsupported directives are refused and morphology-only flags load COMPOUNDFIRST and COMPOUNDLAST, the older names of COMPOUNDBEGIN and COMPOUNDEND, are refused like them until compounding is supported. --- crates/proof/src/affix.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 4ce915afa..77364511f 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -42,7 +42,9 @@ const UNSUPPORTED: &[&str] = &[ "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND", + "COMPOUNDFIRST", "COMPOUNDFLAG", + "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDMIN", "COMPOUNDPERMITFLAG", @@ -1594,6 +1596,30 @@ pub(crate) mod tests { assert!(back.check("km") && !back.check("Km")); } + #[test] + fn morphology_only_flags_leave_words_correct() { + // `SUBSTANDARD` and `LEMMA_PRESENT` only steer morphological generation: the stems and + // rules they mark are still correct. + let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap(); + assert!(d.check("kot") && d.check("koty") && d.check("kotem")); + assert!(d.check("pies") && d.check("piesy") && d.check("piesem")); + } + + #[test] + fn unsupported_directives_are_refused_and_the_rest_load() { + // A dictionary that needs a directive outside this engine fails to load rather than + // accepting the wrong words. + for dir in UNSUPPORTED { + assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); + } + // The directives this engine reads, with no other change to the dictionary. + for line in ["AF 1\nAF ab", "AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); + assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); + } + } + #[test] fn fields_split_on_spaces_and_tabs_only() { // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator. From bb023684f26035a83dcd1de4a6059a3f222848d4 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:01:57 +0200 Subject: [PATCH 14/21] proof: SUBSTANDARD stems are correct but never suggested, as with NOSUGGEST --- crates/proof/src/affix.rs | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 77364511f..4f15bd56e 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -244,6 +244,8 @@ pub struct AffixDict { forbidden: Mask, /// Stems with NOSUGGEST are correct, but none of their forms are suggested. no_suggest: Mask, + /// Stems with SUBSTANDARD are correct too, and likewise never suggested. + substandard: Mask, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, /// Flag char of each bit. @@ -334,6 +336,7 @@ struct Rules { /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option, no_suggest: Option, + substandard: Option, /// The `KEEPCASE` flag. keepcase: Option, iconv: Vec<(String, String)>, @@ -430,9 +433,10 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} - // Morphological aliases, and flags that only matter for morphological generation - // and suggestions: the words they mark are correct. - "AM" | "SUBSTANDARD" | "LEMMA_PRESENT" => {} + // Morphological aliases, and a flag that only matters for morphological generation: + // the words it marks are correct. + "AM" | "LEMMA_PRESENT" => {} + "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), "AF" => { if !af_header && f.len() == 2 && at(1).parse::().is_ok() { af_header = true; @@ -587,6 +591,7 @@ impl AffixDict { flag_chars: flag_chars.clone(), forbidden, no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), + substandard: rules.substandard.map_or(Mask::EMPTY, bit_of), keepcase, iconv: rules.iconv, ignore: rules.ignore.into_boxed_slice(), @@ -800,9 +805,10 @@ impl AffixDict { self.contains_except(&self.normalize(w), Mask::EMPTY) } - /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded). + /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD + /// stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool { - self.contains_except(&self.normalize(w), self.no_suggest) + self.contains_except(&self.normalize(w), self.no_suggest | self.substandard) } /// `w` after the `ICONV` conversions (at each position the longest match wins) and without @@ -912,7 +918,7 @@ impl AffixDict { /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool { - self.check_except(w, self.no_suggest) + self.check_except(w, self.no_suggest | self.substandard) } fn check_except(&self, w: &str, deny: Mask) -> bool { @@ -1079,7 +1085,9 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("KEEPCASE", self.keepcase)] { + for (directive, mask) in + [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] + { if !mask.is_empty() && let Some(c) = self.flag_char(mask) { @@ -1148,11 +1156,12 @@ impl AffixDict { match at(0) { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "KEEPCASE") => { + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => { let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; match directive { "FORBIDDENWORD" => rules.forbidden = Some(c), "NOSUGGEST" => rules.no_suggest = Some(c), + "SUBSTANDARD" => rules.substandard = Some(c), _ => rules.keepcase = Some(c), } } @@ -1597,13 +1606,17 @@ pub(crate) mod tests { } #[test] - fn morphology_only_flags_leave_words_correct() { - // `SUBSTANDARD` and `LEMMA_PRESENT` only steer morphological generation: the stems and - // rules they mark are still correct. + fn substandard_and_lemma_present_words_are_correct() { + // `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD + // stem is never suggested; the stems and rules they mark are still correct. let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n"; let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap(); - assert!(d.check("kot") && d.check("koty") && d.check("kotem")); - assert!(d.check("pies") && d.check("piesy") && d.check("piesem")); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem")); + assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem")); + assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot")); + assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies")); + } } #[test] From 74449b87fb63e707950d68f94f5991330394add7 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:02:51 +0200 Subject: [PATCH 15/21] proof: read count comments, phrase entries and undefined aliases as Hunspell does (#735) Checked against the Hunspell 1.7 command line: - The first AF and ICONV line is the count even with a trailing comment (AF 2 # aliases), so alias numbers no longer shift and ICONV 1 # no longer turns 1 into #. - A .dic entry ends where Hunspell ends it (dic_entry_end), so ad hoc/B is one entry and ad alone is no word. - An undefined alias in a .dic entry reads as no flags: the stem loads without affixes instead of failing the whole dictionary. - IGNORE characters are removed from the appended text only; a stripped text that holds one never matches a stem, as in Hunspell. - FLAG UTF-8 and SET are found anywhere in the .aff, not only in the first 8 KB. Also: normalize borrows the input when no conversion or ignored character applies, the UTF-8 .aff reader counts fields as it goes and builds its error context only on error, and the single-byte flag reader is decode_latin1. --- crates/proof/src/affix.rs | 81 ++++++++++++++++++++++++++++++++---- crates/proof/src/hunspell.rs | 67 +++++++++++++++++------------ 2 files changed, 114 insertions(+), 34 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 4f15bd56e..2d3a7717b 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -266,7 +266,7 @@ pub struct AffixDict { keepcase: Mask, /// `ICONV`: input conversions applied before a lookup, `(from, to)`. iconv: Vec<(String, String)>, - /// `IGNORE`: characters removed from input words (and from the stems and affixes). + /// `IGNORE`: characters removed from input words (and from the stems and appended texts). ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, @@ -361,6 +361,12 @@ impl Rules { .ok_or_else(|| format!("flag alias {written} is not defined")) } + /// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the + /// stem is still a word, without affixes. + fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str { + self.flags(written).unwrap_or("") + } + /// `w` without the `IGNORE` characters. fn strip_ignored(&self, w: &str) -> String { if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } @@ -437,15 +443,17 @@ fn parse_aff(aff: &str) -> Result { // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // trailing comment (`AF 1559 # aliases`). "AF" => { - if !af_header && f.len() == 2 && at(1).parse::().is_ok() { + if !af_header && at(1).parse::().is_ok() { af_header = true; } else { r.aliases.push(at(1).to_string()); } } "ICONV" => { - if !iconv_header && f.len() == 2 && at(1).parse::().is_ok() { + if !iconv_header && at(1).parse::().is_ok() { iconv_header = true; } else if f.len() >= 3 { r.iconv.push((at(1).to_string(), at(2).to_string())); @@ -512,10 +520,11 @@ fn parse_aff(aff: &str) -> Result { } } // Aliases may be declared after the rules that use them; resolve once all are read. + // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that + // holds one never matches a stem (stems lose them too), so the rule never applies. let resolve = |rules: &mut Vec, r: &Rules| -> Result<(), String> { for rule in rules.iter_mut() { rule.4 = r.flags(&rule.4)?.to_string(); - rule.2 = r.strip_ignored(&rule.2); rule.3 = r.strip_ignored(&rule.3); } Ok(()) @@ -538,7 +547,12 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { - let Some(tok) = split_fields(line).next() else { continue }; + // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. + let line = line.trim_start_matches([' ', '\t']); + let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); + if tok.is_empty() { + continue; + } if n == 0 && tok.parse::().is_ok() { continue; } @@ -565,7 +579,7 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } - let flags = rules.flags(&flags).map_err(|e| format!(".dic line {}: {e}", n + 1))?.to_string(); + let flags = rules.dic_flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -814,7 +828,8 @@ impl AffixDict { /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { - if self.iconv.is_empty() && self.ignore.is_empty() { + let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str())); + if !w.contains(&*self.ignore) && !converts(w) { return std::borrow::Cow::Borrowed(w); } let mut out = String::with_capacity(w.len()); @@ -1577,7 +1592,10 @@ pub(crate) mod tests { let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); assert!(!d.contains("házban"), "the stem has a and b, not c"); - assert!(AffixDict::from_hunspell(aff, "1\nház/3\n", &[]).is_err(), "alias 3 is not defined"); + // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes. + let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap(); + assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined"); + assert!(undefined.check("kertok"), "the other entries keep their aliases"); let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); } @@ -1594,6 +1612,53 @@ pub(crate) mod tests { assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); } + #[test] + fn count_lines_may_carry_a_comment() { + // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` + // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`. + let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n"; + let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap(); + assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`"); + assert!(d.check("a1") && !d.check("a#")); + // The UTF-8 reader keeps such a line as it is, so both readers agree on it. + let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap(); + assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}"); + assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok")); + } + + #[test] + fn dic_entries_may_hold_spaces() { + // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. + let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n kot/B 12\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads")); + assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + } + + #[test] + fn ignored_characters_leave_the_stripped_text_alone() { + // Hunspell removes `IGNORE` characters from the appended text, not from the stripped + // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong, + // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line). + let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap(); + assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac")); + let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap(); + assert!(add.check("ac") && add.check("acx")); + for d in [&strip, &add] { + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.check("ac"), d.check("ac")); + } + } + + #[test] + fn normalizing_borrows_when_nothing_changes() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); + assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine"))); + assert_eq!(d.normalize("(fi)ne"), "fine"); + assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot"))); + } + #[test] fn keepcase_stems_match_only_as_written() { let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 934520beb..7152f7a4c 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -224,15 +224,16 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per /// byte); the words, affixes and conditions around them are UTF-8. pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); + // The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic` + // flags are read, as in Hunspell. let directive = |name: &str| { - head.lines().find_map(|l| { - let l = l.trim_start_matches('\u{feff}'); + aff.split(|&b| b == b'\n').find_map(|l| { + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8. - l.strip_prefix(name) - .filter(|rest| rest.starts_with([' ', '\t'])) - .and_then(|rest| rest.split([' ', '\t']).find(|f| !f.is_empty())) - .map(str::to_ascii_uppercase) + l.strip_prefix(name.as_bytes()) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())) + .map(|f| String::from_utf8_lossy(f).to_ascii_uppercase()) }) }; let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); @@ -284,13 +285,9 @@ const FLAG_DIRECTIVES: &[&str] = &[ /// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; -/// One byte per char: a flag field read as single-byte flags. -fn bytes_as_flags(b: &[u8]) -> String { - b.iter().map(|&x| char::from(x)).collect() -} - -/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). -fn text_and_flags(field: &[u8], what: &str) -> Result { +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what` +/// names the line in an error. +fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result { let mut slash = None; let mut i = 0; while let Some(&b) = field.get(i) { @@ -308,8 +305,8 @@ fn text_and_flags(field: &[u8], what: &str) -> Result { Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), None => (field, &[][..]), }; - let mut out = std::str::from_utf8(text).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(); - out.push_str(&bytes_as_flags(flags)); + let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(); + out.push_str(&decode_latin1(flags)); Ok(out) } @@ -353,7 +350,7 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { .and_then(|&i| parts.get(i)) .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) .unwrap_or_default(); - let what = format!(".aff line {}", n + 1); + let what = || format!(".aff line {}", n + 1); if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { out.push_str(&String::from_utf8_lossy(line)); continue; @@ -364,18 +361,23 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); let first_free = comment_field(&dir, field_text); let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); + // The field index of the next non-separator part. + let mut next_field = 0; for (i, p) in parts.iter().enumerate() { - let field = words.iter().position(|&w| w == i); + let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| { + next_field += 1; + next_field - 1 + }); let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); let decoded = match (dir.as_str(), field) { _ if in_comment => String::from_utf8_lossy(p).into_owned(), - ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => bytes_as_flags(p), - ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?, + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?, // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. - ("PFX" | "SFX", Some(1)) => bytes_as_flags(p.get(..1).unwrap_or_default()), - (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p.get(..1).unwrap_or_default()), - _ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(), + ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()), + _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(), }; out.push_str(&decoded); } @@ -433,13 +435,13 @@ fn decode_utf8_dic(dic: &[u8]) -> Result { out.push_str(s); continue; } - let what = format!(".dic line {}", n + 1); + let what = || format!(".dic line {}", n + 1); // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); // morphological fields after it are UTF-8 text. let end = dic_entry_end(line); - out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?); let rest = line.get(end..).unwrap_or_default(); - out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?); } Ok(out) } @@ -522,6 +524,19 @@ mod tests { assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); } + #[test] + fn flag_utf8_counts_anywhere_in_the_aff() { + // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. + let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec(); + for _ in 0..400 { + aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n"); + } + aff.extend_from_slice(b"FLAG UTF-8\r\n"); + assert!(aff.len() > 8192); + let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + } + #[test] fn a_comment_after_flag_utf8_keeps_unicode_flags() { let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap(); From 3051fab79246566278898522978b13d3d951882f Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:09:52 +0200 Subject: [PATCH 16/21] proof: .dic lines and alias numbers read as Hunspell reads them (#735) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checked against the Hunspell 1.7 command line: - Without a tab or a morphological field the whole line is the entry, so a phrase with single-byte flags (tök jó/396 in hu_HU) no longer fails the load or loses its alias; leading blanks belong to the entry. - With an AF table a flag field is the alias its leading digits number (fal/2x is alias 2, lab/b has none); an undefined alias, in a .dic entry or a continuation class, reads as no flags instead of failing the dictionary; a table without its count line is ignored. - A REP count line may carry a comment, like AF and ICONV. normalize copies the input only from its first change on, and SET and FLAG are found in one pass over the .aff. --- crates/proof/src/affix.rs | 103 ++++++++++++++++++++++------------- crates/proof/src/hunspell.rs | 61 ++++++++++++++------- 2 files changed, 107 insertions(+), 57 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 2d3a7717b..3fbb756f6 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -343,28 +343,27 @@ struct Rules { ignore: Vec, /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. aliases: Vec, + /// The `AF` table opened with its count line, so flag fields are alias numbers. A table + /// without one is rejected, as in Hunspell: its lines are ignored. + aliases_on: bool, } impl Rules { - /// Flags as written in a `.dic` entry or a continuation class: an `AF` alias number when the - /// dictionary declares aliases, otherwise the flag characters themselves. - fn flags<'a>(&'a self, written: &'a str) -> Result<&'a str, String> { - if self.aliases.is_empty() || written.is_empty() || !written.bytes().all(|b| b.is_ascii_digit()) { - return Ok(written); + /// Flags as written in a `.dic` entry or a continuation class: the flag characters + /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads + /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: + /// the stem is still a word, the rule still applies, without the flags (Hunspell warns). + fn flags<'a>(&'a self, written: &'a str) -> &'a str { + if !self.aliases_on { + return written; } + let digits = written.bytes().take_while(u8::is_ascii_digit).count(); written - .parse::() - .ok() + .get(..digits) + .and_then(|d| d.parse::().ok()) .and_then(|n| n.checked_sub(1)) .and_then(|i| self.aliases.get(i)) - .map(String::as_str) - .ok_or_else(|| format!("flag alias {written} is not defined")) - } - - /// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the - /// stem is still a word, without affixes. - fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str { - self.flags(written).unwrap_or("") + .map_or("", String::as_str) } /// `w` without the `IGNORE` characters. @@ -446,9 +445,10 @@ fn parse_aff(aff: &str) -> Result { // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a // trailing comment (`AF 1559 # aliases`). "AF" => { - if !af_header && at(1).parse::().is_ok() { + if !af_header { af_header = true; - } else { + r.aliases_on = at(1).parse::().is_ok(); + } else if r.aliases_on { r.aliases.push(at(1).to_string()); } } @@ -479,7 +479,7 @@ fn parse_aff(aff: &str) -> Result { } } "REP" => { - if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + if !rep_header && at(1).parse::().is_ok() { rep_header = true; } else if f.len() >= 3 { r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); @@ -522,16 +522,15 @@ fn parse_aff(aff: &str) -> Result { // Aliases may be declared after the rules that use them; resolve once all are read. // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that // holds one never matches a stem (stems lose them too), so the rule never applies. - let resolve = |rules: &mut Vec, r: &Rules| -> Result<(), String> { + let resolve = |rules: &mut Vec, r: &Rules| { for rule in rules.iter_mut() { - rule.4 = r.flags(&rule.4)?.to_string(); + rule.4 = r.flags(&rule.4).to_string(); rule.3 = r.strip_ignored(&rule.3); } - Ok(()) }; let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); - resolve(&mut pfx, &r)?; - resolve(&mut sfx, &r)?; + resolve(&mut pfx, &r); + resolve(&mut sfx, &r); (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -548,7 +547,7 @@ impl AffixDict { let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. - let line = line.trim_start_matches([' ', '\t']); + // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); if tok.is_empty() { continue; @@ -579,7 +578,7 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } - let flags = rules.dic_flags(&flags).to_string(); + let flags = rules.flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -828,28 +827,31 @@ impl AffixDict { /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { - let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str())); - if !w.contains(&*self.ignore) && !converts(w) { + if self.iconv.is_empty() && self.ignore.is_empty() { return std::borrow::Cow::Borrowed(w); } - let mut out = String::with_capacity(w.len()); + // One pass: the input is copied only from the first change on. + let mut out: Option = None; let mut rest = w; while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); match hit { Some((from, to)) => { - out.extend(to.chars().filter(|c| !self.ignore.contains(c))); + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); rest = rest.get(from.len()..).unwrap_or_default(); } None => { - if !self.ignore.contains(&c) { - out.push(c); + if self.ignore.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); } rest = rest.get(c.len_utf8()..).unwrap_or_default(); } } } - std::borrow::Cow::Owned(out) + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) } /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't @@ -1612,6 +1614,28 @@ pub(crate) mod tests { assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); } + #[test] + fn alias_numbers_are_read_as_hunspell_reads_them() { + // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias + // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags. + let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n"; + let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap(); + assert!(d.check("lab") && !d.check("labq") && !d.check("labs")); + assert!(d.check("falx") && !d.check("fals") && d.check("kuts")); + // A table without its count line is rejected: no alias applies. + let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); + assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // An undefined alias in a continuation class leaves the rule in force, without it. + let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); + assert!(cont.check("haz") && cont.check("hazok")); + } + + #[test] + fn a_rep_count_line_may_carry_a_comment() { + let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap(); + assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]); + } + #[test] fn count_lines_may_carry_a_comment() { // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` @@ -1630,10 +1654,14 @@ pub(crate) mod tests { fn dic_entries_may_hold_spaces() { // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; - let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n kot/B 12\n", &[]).unwrap(); - assert!(d.contains("ad hoc") && d.contains("ad hocs")); - assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads")); + // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the + // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it + // (` lab` is no `lab`). + let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így")); assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + assert!(!d.contains("lab") && !d.contains("labs")); } #[test] @@ -1691,8 +1719,9 @@ pub(crate) mod tests { for dir in UNSUPPORTED { assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); } - // The directives this engine reads, with no other change to the dictionary. - for line in ["AF 1\nAF ab", "AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + // The directives this engine reads, with no other change to the dictionary (`AF` would + // turn the flags of `DIC` into alias numbers; it has tests of its own). + for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 7152f7a4c..d2d26416f 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -224,22 +224,11 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per /// byte); the words, affixes and conditions around them are UTF-8. pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - // The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic` - // flags are read, as in Hunspell. - let directive = |name: &str| { - aff.split(|&b| b == b'\n').find_map(|l| { - let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); - // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8. - l.strip_prefix(name.as_bytes()) - .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) - .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())) - .map(|f| String::from_utf8_lossy(f).to_ascii_uppercase()) - }) - }; - let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); + let (set, flag) = directives(aff); + let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - if matches!(directive("FLAG").as_deref(), Some("UTF-8" | "UTF8")) { + if matches!(flag.as_deref(), Some("UTF-8" | "UTF8")) { let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); } @@ -254,6 +243,32 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass +/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read, +/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8. +fn directives(aff: &[u8]) -> (Option, Option) { + let (mut set, mut flag) = (None, None); + for l in aff.split(|&b| b == b'\n') { + if set.is_some() && flag.is_some() { + break; + } + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); + let slot = match l.first() { + Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]), + Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]), + _ => continue, + }; + let arg = l + .strip_prefix(slot.1) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())); + if let Some(a) = arg { + *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase()); + } + } + (set, flag) +} + /// Directives whose one argument is a flag. const FLAG_DIRECTIVES: &[&str] = &[ "CIRCUMFIX", @@ -406,7 +421,8 @@ fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { /// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the /// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever -/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the first space. +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line +/// (`kot/B 12` has the flags `B 12`, as in Hunspell). pub(crate) fn dic_entry_end(line: &[u8]) -> usize { let blank = |b: &u8| matches!(b, b' ' | b'\t'); let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); @@ -415,10 +431,10 @@ pub(crate) fn dic_entry_end(line: &[u8]) -> usize { (Some(m), Some(t)) => m.min(t), (m, t) => match m.or(t) { Some(e) => e, - None => line.iter().position(|&b| b == b' ').unwrap_or(line.len()), + None => line.len(), }, }; - // Spaces before the morphological field aren't part of the entry. + // Blanks before the morphological field or the line end aren't part of the entry. line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) } @@ -495,10 +511,10 @@ mod tests { fn utf8_dictionaries_keep_single_byte_flags() { // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. - let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 1\nAF \xd5a # 1\nSFX \xd5 Y 1\nSFX \xd5 0 ok/\xe9 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); let (a, d) = decode(aff, dic).unwrap(); - assert!(a.contains("AF \u{d5}a # 1") && a.contains("SFX \u{d5} 0 ok/\u{e9} . 7")); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7")); assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); @@ -578,12 +594,17 @@ mod tests { assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); - assert_eq!(end("word/A 123"), "word/A", "neither: the first space"); + assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell"); + assert_eq!(end("ad hoc/B "), "ad hoc/B"); assert_eq!(end("word"), "word"); assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + // Without a tab or a morphological field the whole line is the entry: its byte flags + // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`). + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\n"); } #[test] From a47412aef9faec369bf62ef2bb4eb120b0a5d442 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:46:25 +0100 Subject: [PATCH 17/21] proof: an AF count below 1 rejects the table, flags stay literal (#735) --- crates/proof/src/affix.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 3fbb756f6..7d4fc1821 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -447,7 +447,8 @@ fn parse_aff(aff: &str) -> Result { "AF" => { if !af_header { af_header = true; - r.aliases_on = at(1).parse::().is_ok(); + // A count below 1 rejects the table too: flags stay literal (`AF 0`). + r.aliases_on = at(1).parse::().is_ok_and(|n| n > 0); } else if r.aliases_on { r.aliases.push(at(1).to_string()); } @@ -1625,6 +1626,9 @@ pub(crate) mod tests { // A table without its count line is rejected: no alias applies. let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // `AF 0` is rejected the same way: `lab/B` keeps its literal flag. + let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap(); + assert!(zero.check("lab") && zero.check("labs")); // An undefined alias in a continuation class leaves the rule in force, without it. let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); assert!(cont.check("haz") && cont.check("hazok")); From e4765c999d723d3dde11ddb422465e64153c1954 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:13:24 +0100 Subject: [PATCH 18/21] proof: numeric and long flags keep a UTF-8 .aff as text (#735) --- crates/proof/src/hunspell.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index d2d26416f..1afd903cd 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -219,16 +219,18 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R /// and Windows-1251 the ones older Russian dictionaries use). /// -/// Under a UTF-8 `SET` without `FLAG UTF-8`, flags are still single bytes, as in Hunspell: the -/// Hungarian dictionary writes its affix flags and flag aliases as bytes above 0x7F that aren't -/// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per -/// byte); the words, affixes and conditions around them are UTF-8. +/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes, +/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above +/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, +/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other +/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps +/// its number (the Turkish dictionary's `FLAG num`). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let (set, flag) = directives(aff); let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - if matches!(flag.as_deref(), Some("UTF-8" | "UTF8")) { + if flag.as_deref().is_some_and(|f| f != "DEFAULT") { let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); } @@ -540,6 +542,24 @@ mod tests { assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); } + #[test] + fn numeric_and_long_flags_stay_text() { + // `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a + // number with a non-ASCII rule or comment keeps every digit. + let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n"; + let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap(); + assert_eq!(a, aff); + assert_eq!(d, "1\nkitap/1234\n"); + let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap(); + assert!(tr.contains("kitapları") && tr.contains("kitap")); + let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n"; + assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long); + assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8"); + // `FLAG DEFAULT` is the single-byte default. + let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap(); + assert_eq!(d, "1\nkert/\u{e9}\n"); + } + #[test] fn flag_utf8_counts_anywhere_in_the_aff() { // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. From fa657a1c41ae49066e966e8b99f79706e97e27a6 Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:21:39 +0100 Subject: [PATCH 19/21] proof: U+0085 and U+00A0 are flags in one-flag directives too (#735) --- crates/proof/src/affix.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 7d4fc1821..c966cf131 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -382,7 +382,9 @@ fn split_fields(line: &str) -> impl Iterator { fn one_flag(arg: &str, dir: &str, n: usize) -> Result { let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => Ok(c), + // Only the field separators of `split_fields` can't be a flag: a flag byte read as + // U+0085 or U+00A0 is one. + (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c), _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), } } @@ -1739,6 +1741,13 @@ pub(crate) mod tests { assert!(d.contains("koty") && d.contains("kotem")); let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); assert!(back.contains("koty") && back.contains("kotem")); + // The same bytes as one-flag directives, as Hunspell reads them. + let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kots") && !dict.check("rossz")); + assert!(dict.check("pH") && !dict.check("PH")); + } } #[test] From 0d41b183010c55e87fb23c9ab37c6dda72e51e05 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Thu, 29 Oct 2026 15:38:01 -0200 Subject: [PATCH 20/21] Engine tests: the dictionaries-folder test checks each reported file again The shared dictionary slot reports no file while a dictionary given with install_dictionary is in use, so this test had accepted null alongside the folder's file, for installs from tests running at the same time. That also let it pass if file.options stopped reporting the folder's file at all. The tests' install helpers and this test now take turns on one lock, and each language must report the folder's file again. An install from before the test doesn't matter: changing the folder reloads every slot. --- crates/engine/src/tests.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index aa1d995b6..267934ad4 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -8038,9 +8038,18 @@ fn next_footnote_skips_endnotes() { assert_eq!(s.sel.focus.off, 9); } +/// Tests that install a test dictionary wait for the dictionaries-folder test, which changes the +/// folder and then reads what each language reports: an install in between reports no file. +static DICTIONARIES: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn dictionaries_turn() -> std::sync::MutexGuard<'static, ()> { + DICTIONARIES.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// A tiny synthetic Polish Hunspell dictionary (written for these tests). Real dictionaries are /// installed on the system, never bundled; every Polish test installs this same one. fn polish_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::pl::install_dictionary(POLISH_TEST_AFF, POLISH_TEST_DIC).unwrap(); } @@ -8050,6 +8059,7 @@ const POLISH_TEST_DIC: &str = "12\npłynie\nrzeka\na\nto\nwiem\nże\ntak\nw\nog /// A tiny synthetic Portuguese Hunspell dictionary (written for these tests). Real dictionaries /// are installed on the system, never bundled; every Portuguese test installs this same one. fn portuguese_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::pt::install_dictionary(PORTUGUESE_TEST_AFF, PORTUGUESE_TEST_DIC).unwrap(); } @@ -8063,6 +8073,7 @@ const PORTUGUESE_TEST_DIC: &str = "11\nsaudade\ncoração\ne\nele\ndisse\nnão\n /// running meanwhile see the same words; it stays set for the rest of the run for that reason. #[test] fn dictionaries_folder_option_is_saved_searched_and_reported() { + let _turn = dictionaries_turn(); let dir = std::env::temp_dir().join(format!("wc-engine-dictionaries-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); for (name, aff, dic) in [ @@ -8093,12 +8104,10 @@ fn dictionaries_folder_option_is_saved_searched_and_reported() { assert_eq!(r["dictionariesFolder"], json!(dir.to_string_lossy())); let dicts = r["dictionaries"].as_array().unwrap(); let file = |lang: &str| dicts.iter().find(|d| d["language"] == lang).map(|d| d["file"].clone()); - // A language reports the folder's file, or none while a dictionary another test gave with - // install_dictionary() is in use (those tests run concurrently); the folder is searched first - // either way. + // Each language reports the folder's file, which is searched first. let reported = |got: Option<&serde_json::Value>, name: &str| { let want = dir.join(format!("{name}.dic")); - assert!(got.is_some_and(|f| f.is_null() || *f == json!(want.to_string_lossy())), "{name}: {dicts:?}"); + assert_eq!(got, Some(&json!(want.to_string_lossy())), "{name}: {dicts:?}"); assert_eq!(wordcraft_proof::hunspell::find(&[name]).map(|(_, dic)| dic), Some(want)); }; reported(file("pl").as_ref(), "pl_PL"); @@ -8158,6 +8167,7 @@ fn portuguese_proofing_through_the_review_commands() { } fn indonesian_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::id::install_dictionary(INDONESIAN_TEST_AFF, INDONESIAN_TEST_DIC).unwrap(); } @@ -8329,6 +8339,7 @@ const URUGUAYAN_TEST_DIC: &str = "10\nesto\nes\nuna\nprueba/S\ndel\ncorrector\nt fn spanish_test_dictionaries() { use wordcraft_proof::ProofLang; + let _turn = dictionaries_turn(); for (lang, dic) in [(ProofLang::Es, SPANISH_TEST_DIC), (ProofLang::EsUy, URUGUAYAN_TEST_DIC)] { wordcraft_proof::es::install_dictionary(lang, SPANISH_TEST_AFF, dic).unwrap(); } @@ -8457,6 +8468,7 @@ const ITALIAN_TEST_AFF: &str = "SET UTF-8\nTRY aeiortnsclcaffèéè\nMAP 2\nMAP const ITALIAN_TEST_DIC: &str = "6\ncasa\ncaffè\nroma\ndisse\ne\nper\n"; fn italian_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::it::install_dictionary(ITALIAN_TEST_AFF, ITALIAN_TEST_DIC).unwrap(); } From 78f3089b19f8fb842aa6447a2a63f354ff8b3a36 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 10:46:06 +0000 Subject: [PATCH 21/21] =?UTF-8?q?#5000:=20review=20=E2=80=94=20a=20KEEPCAS?= =?UTF-8?q?E=20word=20is=20suggested=20in=20its=20dictionary=20case,=20nev?= =?UTF-8?q?er=20re-cased=20into=20a=20flagged=20form?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Kmm` with `km/K` suggested `Km`, which the checker flags; `Km` itself got no suggestion. A suggestion whose dictionary form is only correct through a KEEPCASE stem now keeps that case (`km`). Other stems are re-cased as before (`MIXCASX` → `MIXCASE`). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 6 ++++++ crates/proof/src/affix_suggest.rs | 21 +++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c966cf131..cb4621210 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -941,6 +941,12 @@ impl AffixDict { self.check_except(w, self.no_suggest | self.substandard) } + /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case + /// alone (`km` from `km/K`, which `Km` and `KM` are not). + pub(crate) fn case_locked(&self, w: &str) -> bool { + self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase) + } + fn check_except(&self, w: &str, deny: Mask) -> bool { let w = self.normalize(w); if self.contains_except(&w, deny) { diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 3d7cef643..1456f81c7 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -297,11 +297,13 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); let mut out: Vec = Vec::new(); for (text, _) in found { - let t = match case { + let recased = match case { Casing::Initial => capitalize(&text), Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, + _ => text.clone(), }; + // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`. + let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased }; if t != word && !out.contains(&t) { out.push(t); } @@ -521,6 +523,21 @@ mod tests { assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); } + /// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged + /// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form. + #[test] + fn keepcase_suggestions_are_not_recased_into_errors() { + let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap(); + assert!(!d.check("Km") && !d.check("KM")); + for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] { + let got = suggest_in(&d, word, 4, &Tuning::DEFAULT); + assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}"); + assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}"); + } + assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::::new()); + assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies")); + } + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY /// line, and a split rule decides which missing spaces are offered. #[test]