diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 6cc770c64..506033cb1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -8472,9 +8472,18 @@ fn next_footnote_skips_endnotes() { assert_eq!(s.sel.focus.off, 9); } +/// Tests that install a test dictionary wait for the dictionaries-folder test, which changes the +/// folder and then reads what each language reports: an install in between reports no file. +static DICTIONARIES: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn dictionaries_turn() -> std::sync::MutexGuard<'static, ()> { + DICTIONARIES.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// A tiny synthetic Polish Hunspell dictionary (written for these tests). Real dictionaries are /// installed on the system, never bundled; every Polish test installs this same one. fn polish_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::pl::install_dictionary(POLISH_TEST_AFF, POLISH_TEST_DIC).unwrap(); } @@ -8484,6 +8493,7 @@ const POLISH_TEST_DIC: &str = "12\npłynie\nrzeka\na\nto\nwiem\nże\ntak\nw\nog /// A tiny synthetic Portuguese Hunspell dictionary (written for these tests). Real dictionaries /// are installed on the system, never bundled; every Portuguese test installs this same one. fn portuguese_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::pt::install_dictionary(PORTUGUESE_TEST_AFF, PORTUGUESE_TEST_DIC).unwrap(); } @@ -8497,6 +8507,7 @@ const PORTUGUESE_TEST_DIC: &str = "11\nsaudade\ncoração\ne\nele\ndisse\nnão\n /// running meanwhile see the same words; it stays set for the rest of the run for that reason. #[test] fn dictionaries_folder_option_is_saved_searched_and_reported() { + let _turn = dictionaries_turn(); let dir = std::env::temp_dir().join(format!("wc-engine-dictionaries-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); for (name, aff, dic) in [ @@ -8527,14 +8538,20 @@ fn dictionaries_folder_option_is_saved_searched_and_reported() { assert_eq!(r["dictionariesFolder"], json!(dir.to_string_lossy())); let dicts = r["dictionaries"].as_array().unwrap(); let file = |lang: &str| dicts.iter().find(|d| d["language"] == lang).map(|d| d["file"].clone()); - assert_eq!(file("pl"), Some(json!(dir.join("pl_PL.dic").to_string_lossy()))); - assert_eq!(file("pt"), Some(json!(dir.join("pt_BR.dic").to_string_lossy()))); - assert_eq!(file("id"), Some(json!(dir.join("id_ID.dic").to_string_lossy()))); + // Each language reports the folder's file, which is searched first. + let reported = |got: Option<&serde_json::Value>, name: &str| { + let want = dir.join(format!("{name}.dic")); + assert_eq!(got, Some(&json!(want.to_string_lossy())), "{name}: {dicts:?}"); + assert_eq!(wordcraft_proof::hunspell::find(&[name]).map(|(_, dic)| dic), Some(want)); + }; + reported(file("pl").as_ref(), "pl_PL"); + reported(file("pt").as_ref(), "pt_BR"); + reported(file("id").as_ref(), "id_ID"); // Each Spanish variety reports its own file (Spain, Mexico, Latin America, Uruguay, in order). let spanish: Vec = dicts.iter().filter(|d| d["language"] == "es").map(|d| d["file"].clone()).collect(); assert_eq!(spanish.len(), 4); - assert_eq!(spanish.first(), Some(&json!(dir.join("es_ES.dic").to_string_lossy()))); - assert_eq!(spanish.last(), Some(&json!(dir.join("es_UY.dic").to_string_lossy()))); + reported(spanish.first(), "es_ES"); + reported(spanish.last(), "es_UY"); // Polish text is checked against the folder's dictionary. run(&mut s, "document.setText", json!({"text": "rzeka płynie żeka"})); run(&mut s, "select.all", json!({})); @@ -8584,6 +8601,7 @@ fn portuguese_proofing_through_the_review_commands() { } fn indonesian_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::id::install_dictionary(INDONESIAN_TEST_AFF, INDONESIAN_TEST_DIC).unwrap(); } @@ -8755,6 +8773,7 @@ const URUGUAYAN_TEST_DIC: &str = "10\nesto\nes\nuna\nprueba/S\ndel\ncorrector\nt fn spanish_test_dictionaries() { use wordcraft_proof::ProofLang; + let _turn = dictionaries_turn(); for (lang, dic) in [(ProofLang::Es, SPANISH_TEST_DIC), (ProofLang::EsUy, URUGUAYAN_TEST_DIC)] { wordcraft_proof::es::install_dictionary(lang, SPANISH_TEST_AFF, dic).unwrap(); } @@ -8883,6 +8902,7 @@ const ITALIAN_TEST_AFF: &str = "SET UTF-8\nTRY aeiortnsclcaffèéè\nMAP 2\nMAP const ITALIAN_TEST_DIC: &str = "6\ncasa\ncaffè\nroma\ndisse\ne\nper\n"; fn italian_test_dictionary() { + let _turn = dictionaries_turn(); wordcraft_proof::it::install_dictionary(ITALIAN_TEST_AFF, ITALIAN_TEST_DIC).unwrap(); } diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 4ec068466..cb4621210 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -11,10 +11,14 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //! -//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), -//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross -//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), -//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding, +//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`), +//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and +//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes +//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the +//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` +//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` +//! and `REP`. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding, //! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and @@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ - "AF", - "AM", "CHECKSHARPS", "CIRCUMFIX", "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND", + "COMPOUNDFIRST", "COMPOUNDFLAG", + "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDMIN", "COMPOUNDPERMITFLAG", "COMPOUNDRULE", "FULLSTRIP", - "ICONV", - "IGNORE", - "KEEPCASE", - "LEMMA_PRESENT", "NEEDAFFIX", "OCONV", "ONLYINCOMPOUND", "PSEUDOROOT", - "SUBSTANDARD", ]; +/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary +/// uses 158). +const MAX_FLAGS: usize = 256; + +/// A set of flags, one bit per distinct flag character of the dictionary. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +struct Mask([u64; MAX_FLAGS / 64]); + +impl Mask { + const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]); + + /// The set holding flag number `i` alone (empty when `i` is out of range). + fn bit(i: usize) -> Mask { + let mut m = Mask::EMPTY; + if let Some(w) = m.0.get_mut(i / 64) { + *w = 1 << (i % 64); + } + m + } + + /// The number of the lowest flag in the set, if any. + fn first(self) -> Option { + self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize) + } + + fn is_empty(self) -> bool { + self.0.iter().all(|&w| w == 0) + } + + /// Do the two sets share a flag? + fn meets(self, other: Mask) -> bool { + !(self & other).is_empty() + } + + /// Does this set hold every flag of `other`? + fn covers(self, other: Mask) -> bool { + (self & other) == other + } +} + +impl std::ops::BitAnd for Mask { + type Output = Mask; + fn bitand(mut self, o: Mask) -> Mask { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a &= b; + } + self + } +} + +impl std::ops::BitOr for Mask { + type Output = Mask; + fn bitor(mut self, o: Mask) -> Mask { + self |= o; + self + } +} + +impl std::ops::BitOrAssign for Mask { + fn bitor_assign(&mut self, o: Mask) { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a |= b; + } + } +} + /// One character class of a rule's condition. #[derive(Clone, Debug, PartialEq, Eq)] enum Cond { @@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18; #[derive(Clone, Debug)] struct Rule { /// Bit of the rule's flag in the stems' flag masks. - bit: u128, + bit: Mask, flag: char, cross: bool, strip: Box, add: Box, /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. - cont: u128, + cont: Mask, cond: Box<[Cond]>, cond_text: Box, } @@ -174,11 +239,13 @@ pub struct AffixDict { /// Per stem: index into `sets`. flags: Vec, /// Distinct flag masks. - sets: Vec, + sets: Vec, /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). - forbidden: u128, + forbidden: Mask, /// Stems with NOSUGGEST are correct, but none of their forms are suggested. - no_suggest: u128, + no_suggest: Mask, + /// Stems with SUBSTANDARD are correct too, and likewise never suggested. + substandard: Mask, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, /// Flag char of each bit. @@ -195,6 +262,12 @@ pub struct AffixDict { key: String, map: Vec>, rep: Vec<(String, String)>, + /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none). + keepcase: Mask, + /// `ICONV`: input conversions applied before a lookup, `(from, to)`. + iconv: Vec<(String, String)>, + /// `IGNORE`: characters removed from input words (and from the stems and appended texts). + ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, } @@ -263,6 +336,57 @@ struct Rules { /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option, no_suggest: Option, + substandard: Option, + /// The `KEEPCASE` flag. + keepcase: Option, + iconv: Vec<(String, String)>, + ignore: Vec, + /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. + aliases: Vec, + /// The `AF` table opened with its count line, so flag fields are alias numbers. A table + /// without one is rejected, as in Hunspell: its lines are ignored. + aliases_on: bool, +} + +impl Rules { + /// Flags as written in a `.dic` entry or a continuation class: the flag characters + /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads + /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: + /// the stem is still a word, the rule still applies, without the flags (Hunspell warns). + fn flags<'a>(&'a self, written: &'a str) -> &'a str { + if !self.aliases_on { + return written; + } + let digits = written.bytes().take_while(u8::is_ascii_digit).count(); + written + .get(..digits) + .and_then(|d| d.parse::().ok()) + .and_then(|n| n.checked_sub(1)) + .and_then(|i| self.aliases.get(i)) + .map_or("", String::as_str) + } + + /// `w` without the `IGNORE` characters. + fn strip_ignored(&self, w: &str) -> String { + if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } + } +} + +/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a +/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag. +fn split_fields(line: &str) -> impl Iterator { + line.split([' ', '\t']).filter(|f| !f.is_empty()) +} + +/// The one flag character a directive (`FORBIDDENWORD w`) names. +fn one_flag(arg: &str, dir: &str, n: usize) -> Result { + let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); + match (it.next(), it.next()) { + // Only the field separators of `split_fields` can't be a flag: a flag byte read as + // U+0085 or U+00A0 is one. + (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c), + _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), + } } /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. @@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result { let mut cross: HashMap<(bool, char), bool> = HashMap::new(); let mut map_header = false; let mut rep_header = false; + let mut af_header = false; + let mut iconv_header = false; for (n, line) in aff.lines().enumerate() { - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let Some(&dir) = f.first() else { continue }; if dir.starts_with('#') { continue; @@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + // Morphological aliases, and a flag that only matters for morphological generation: + // the words it marks are correct. + "AM" | "LEMMA_PRESENT" => {} + "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // trailing comment (`AF 1559 # aliases`). + "AF" => { + if !af_header { + af_header = true; + // A count below 1 rejects the table too: flags stay literal (`AF 0`). + r.aliases_on = at(1).parse::().is_ok_and(|n| n > 0); + } else if r.aliases_on { + r.aliases.push(at(1).to_string()); + } + } + "ICONV" => { + if !iconv_header && at(1).parse::().is_ok() { + iconv_header = true; + } else if f.len() >= 3 { + r.iconv.push((at(1).to_string(), at(2).to_string())); + } + } + "IGNORE" => r.ignore = at(1).chars().collect(), + "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work // with the one-character parsing here; only the multi-character encodings differ. "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } "FLAG" => {} - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - r.forbidden = Some(c); - } else { - r.no_suggest = Some(c); - } - } - _ => return Err(format!("line {}: {directive} needs one flag character", n + 1)), - } - } + "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?), + "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), "TRY" => r.try_chars = cap_try(at(1)), "KEY" => r.key = at(1).to_string(), "MAP" => { @@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result { } } "REP" => { - if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + if !rep_header && at(1).parse::().is_ok() { rep_header = true; } else if f.len() >= 3 { r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); @@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result { _ => {} } } + // Aliases may be declared after the rules that use them; resolve once all are read. + // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that + // holds one never matches a stem (stems lose them too), so the rule never applies. + let resolve = |rules: &mut Vec, r: &Rules| { + for rule in rules.iter_mut() { + rule.4 = r.flags(&rule.4).to_string(); + rule.3 = r.strip_ignored(&rule.3); + } + }; + let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); + resolve(&mut pfx, &r); + resolve(&mut sfx, &r); + (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -397,7 +549,12 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { - let Some(tok) = line.split_whitespace().next() else { continue }; + // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. + // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). + let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); + if tok.is_empty() { + continue; + } if n == 0 && tok.parse::().is_ok() { continue; } @@ -420,9 +577,11 @@ impl AffixDict { word.push(c); } } + let word = rules.strip_ignored(&word); if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } + let flags = rules.flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -434,11 +593,12 @@ impl AffixDict { flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); flag_chars.sort_unstable(); flag_chars.dedup(); - if flag_chars.len() > 128 { - return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len())); + if flag_chars.len() > MAX_FLAGS { + return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); } - let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i); - let forbidden = rules.forbidden.map_or(0, bit_of); + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit); + let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of); + let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); // Stems sorted by bytes; homonyms (the same word twice) are merged. stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); let mut d = AffixDict { @@ -446,7 +606,11 @@ impl AffixDict { offs: Vec::with_capacity(stems.len() + 1), flag_chars: flag_chars.clone(), forbidden, - no_suggest: rules.no_suggest.map_or(0, bit_of), + no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), + substandard: rules.substandard.map_or(Mask::EMPTY, bit_of), + keepcase, + iconv: rules.iconv, + ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, key: rules.key, map: rules.map, @@ -455,16 +619,16 @@ impl AffixDict { ..AffixDict::default() }; d.offs.push(0); - let mut set_ids: HashMap = HashMap::new(); + let mut set_ids: HashMap = HashMap::new(); let mut i = 0; while i < stems.len() { let Some((word, _)) = stems.get(i) else { break }; - let mut mask = 0u128; + let mut mask = Mask::EMPTY; let mut j = i; while let Some((w, f)) = stems.get(j) && w == word { - mask |= f.chars().map(bit_of).fold(0, |a, b| a | b); + mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); j += 1; } let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; @@ -478,7 +642,7 @@ impl AffixDict { i = j; } let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { - let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c)); + let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) }; d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; @@ -493,7 +657,7 @@ impl AffixDict { // Each (appended, stripped) pair's group, found by hash rather than by scanning the // appended string's groups: a dictionary with many strips per suffix stays linear. let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); - let mut by_flag: Vec> = vec![Vec::new(); 128]; + let mut by_flag: Vec> = vec![Vec::new(); MAX_FLAGS]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; @@ -504,9 +668,7 @@ impl AffixDict { groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); } } - if r.bit != 0 - && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) - { + if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { list.push(i); } } @@ -551,12 +713,12 @@ impl AffixDict { } /// The flag characters of `mask`, in flag order (how the compact format stores flags). - fn flag_chars_for(&self, mask: u128) -> String { - self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect() + fn flag_chars_for(&self, mask: Mask) -> String { + self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() } /// The character of `mask` when it holds exactly one flag. - fn flag_char(&self, mask: u128) -> Option { + fn flag_char(&self, mask: Mask) -> Option { let s = self.flag_chars_for(mask); let mut it = s.chars(); match (it.next(), it.next()) { @@ -589,11 +751,12 @@ impl AffixDict { } /// Flag mask of a stem, if `w` is one. - fn stem_flags(&self, w: &str) -> Option { - self.stem_flags_excluding(w, self.forbidden) + fn stem_flags(&self, w: &str) -> Option { + self.stem_flags_except(w, Mask::EMPTY) } - fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option { + /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`. + fn stem_flags_except(&self, w: &str, deny: Mask) -> Option { if self.table.is_empty() { return None; } @@ -607,8 +770,9 @@ impl AffixDict { if self.word(t as usize) == Some(w) { let id = *self.flags.get(t as usize)?; let mask = self.sets.get(usize::from(id)).copied(); - // All derivations use this lookup: exclusions apply to stems and their forms. - return mask.filter(|m| m & excluded == 0); + // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, + // and (this is the only stem lookup) no affixed form is built on it either. + return mask.filter(|m| !m.meets(self.forbidden | deny)); } slot = (slot + 1) & mask; } @@ -621,8 +785,8 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one mask of their flags and /// followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { + fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool { + if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { return true; } if depth == 0 { @@ -641,9 +805,9 @@ impl AffixDict { .rules .iter() .filter_map(|&i| self.suffixes.get(i as usize)) - .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) - .fold(0u128, |m, r| m | r.bit); - if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true)) + .fold(Mask::EMPTY, |m, r| m | r.bit); + if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { return true; } } @@ -651,21 +815,55 @@ impl AffixDict { false } - /// Is `w` exactly (case and all) a stem or an affixed form of one? + /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE` + /// apply first. pub fn contains(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden) + self.contains_except(&self.normalize(w), Mask::EMPTY) } - /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded). + /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD + /// stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden | self.no_suggest) + self.contains_except(&self.normalize(w), self.no_suggest | self.substandard) } - fn contains_excluding(&self, w: &str, excluded: u128) -> bool { + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without + /// the `IGNORE` characters. + fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { + if self.iconv.is_empty() && self.ignore.is_empty() { + return std::borrow::Cow::Borrowed(w); + } + // One pass: the input is copied only from the first change on. + let mut out: Option = None; + let mut rest = w; + while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); + let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); + match hit { + Some((from, to)) => { + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); + rest = rest.get(from.len()..).unwrap_or_default(); + } + None => { + if self.ignore.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); + } + rest = rest.get(c.len_utf8()..).unwrap_or_default(); + } + } + } + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) + } + + /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't + /// count (case variants of `KEEPCASE` stems). + fn contains_except(&self, w: &str, deny: Mask) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) { + if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { return true; } let mut root = String::with_capacity(w.len() + 8); @@ -680,13 +878,13 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) { + if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { return true; } - if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) { + if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { return true; } - if r.cross && self.suffixed(&root, r.bit, excluded) { + if r.cross && self.suffixed(&root, r.bit, deny) { return true; } } @@ -695,7 +893,7 @@ impl AffixDict { /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must /// carry those flags too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool { + fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { let mut root = String::with_capacity(w.len() + 8); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. @@ -710,18 +908,18 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_excluding(&root, excluded) - && f & need == need - && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)) + if let Some(f) = self.stem_flags_except(&root, deny) + && f.covers(need) + && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. - if need == 0 { - let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); - if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + if need.is_empty() { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit); + if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { return true; } } @@ -732,26 +930,35 @@ impl AffixDict { /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals /// — in lower case (and capitalized), since a sentence may start with any word and headings - /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a + /// `KEEPCASE` stem matches only as written. pub fn check(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden) + self.check_except(w, Mask::EMPTY) } /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden | self.no_suggest) + self.check_except(w, self.no_suggest | self.substandard) } - fn check_excluding(&self, w: &str, excluded: u128) -> bool { - if self.contains_excluding(w, excluded) { + /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case + /// alone (`km` from `km/K`, which `Km` and `KM` are not). + pub(crate) fn case_locked(&self, w: &str) -> bool { + self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase) + } + + fn check_except(&self, w: &str, deny: Mask) -> bool { + let w = self.normalize(w); + if self.contains_except(&w, deny) { return true; } - match casing(w) { + let deny = deny | self.keepcase; + match casing(&w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded), + Casing::Initial => self.contains_except(&w.to_lowercase(), deny), Casing::Upper => { let lower = w.to_lowercase(); - self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded) + self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) } } } @@ -766,11 +973,11 @@ impl AffixDict { let mut cross_sfx = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once. - let mut chain: Vec<(String, u128)> = Vec::new(); - let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut chain: Vec<(String, Mask)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if f & r.bit == 0 || !r.cond_matches(stem, true) { + if !f.meets(r.bit) || !r.cond_matches(stem, true) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -778,7 +985,7 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 && queued.insert((form.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); @@ -804,7 +1011,7 @@ impl AffixDict { } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 && queued.insert((next.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); @@ -815,7 +1022,7 @@ impl AffixDict { } } for r in &self.prefixes { - if f & r.bit == 0 || !r.cond_matches(stem, false) { + if !f.meets(r.bit) || !r.cond_matches(stem, false) { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; @@ -839,11 +1046,11 @@ impl AffixDict { } /// The suffix rules carrying any flag of `mask`, in rule order. - fn suffixes_with(&self, mask: u128) -> Vec { + fn suffixes_with(&self, mask: Mask) -> Vec { let mut rules: Vec = Vec::new(); let mut lists = 0; for (bit, list) in self.sfx_by_flag.iter().enumerate() { - if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + if mask.meets(Mask::bit(bit)) && !list.is_empty() { rules.extend_from_slice(list); lists += 1; } @@ -904,13 +1111,21 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] { - if mask != 0 + for (directive, mask) in + [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] + { + if !mask.is_empty() && let Some(c) = self.flag_char(mask) { out.push_str(&format!("{directive} {c}\n")); } } + for (a, b) in &self.iconv { + out.push_str(&format!("ICONV {a} {b}\n")); + } + if !self.ignore.is_empty() { + out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); + } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules { @@ -934,8 +1149,8 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0); - if mask != 0 { + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default(); + if !mask.is_empty() { raw.push(b'/'); raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); } @@ -962,26 +1177,24 @@ impl AffixDict { comments.push(c.to_string()); continue; } - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - "MAP" => rules.map.push(at(1).chars().collect()), - "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars(); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - rules.forbidden = Some(c); - } else { - rules.no_suggest = Some(c); - } - } - _ => return Err(format!("affix dictionary: bad {directive} in `{line}`")), + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => { + let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; + match directive { + "FORBIDDENWORD" => rules.forbidden = Some(c), + "NOSUGGEST" => rules.no_suggest = Some(c), + "SUBSTANDARD" => rules.substandard = Some(c), + _ => rules.keepcase = Some(c), } } + "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), + "IGNORE" => rules.ignore = at(1).chars().collect(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { let mut fc = at(1).chars(); let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; @@ -1223,7 +1436,7 @@ pub(crate) mod tests { #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin - // letter, the digits and eight accented letters: 70 flags, 64 < n <= 128. + // letter, the digits and eight accented letters: 70 flags, more than 64. let mut flags = String::new(); let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); let chars: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); @@ -1239,6 +1452,24 @@ pub(crate) mod tests { assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); } + #[test] + fn more_than_128_distinct_flags_load() { + // The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up. + let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect(); + let mut aff = String::from("SET UTF-8\n"); + for (i, c) in flags.chars().enumerate() { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n")); + } + let half: String = flags.chars().step_by(2).collect(); + let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap(); + assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130")); + assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("szóx198") && !back.contains("szóx199")); + let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); + assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err()); + } + #[test] fn forbidden_words_are_rejected() { // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying @@ -1364,6 +1595,167 @@ pub(crate) mod tests { assert!(d.contains("kot") && d.contains("koty")); } + #[test] + fn flag_aliases_stand_for_flag_sets() { + // `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags. + // Morphological aliases (`AM`) and the fields after a stem are skipped. + let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n"; + let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); + assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); + assert!(!d.contains("házban"), "the stem has a and b, not c"); + // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes. + let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap(); + assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined"); + assert!(undefined.check("kertok"), "the other entries keep their aliases"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); + } + + #[test] + fn iconv_and_ignore_normalize_input() { + // Ligatures read as their letters; `IGNORE` characters vanish from input and stems. + let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap(); + assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine")); + assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)")); + assert!(!d.check("finer")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); + } + + #[test] + fn alias_numbers_are_read_as_hunspell_reads_them() { + // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias + // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags. + let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n"; + let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap(); + assert!(d.check("lab") && !d.check("labq") && !d.check("labs")); + assert!(d.check("falx") && !d.check("fals") && d.check("kuts")); + // A table without its count line is rejected: no alias applies. + let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); + assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // `AF 0` is rejected the same way: `lab/B` keeps its literal flag. + let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap(); + assert!(zero.check("lab") && zero.check("labs")); + // An undefined alias in a continuation class leaves the rule in force, without it. + let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); + assert!(cont.check("haz") && cont.check("hazok")); + } + + #[test] + fn a_rep_count_line_may_carry_a_comment() { + let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap(); + assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]); + } + + #[test] + fn count_lines_may_carry_a_comment() { + // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` + // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`. + let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n"; + let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap(); + assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`"); + assert!(d.check("a1") && !d.check("a#")); + // The UTF-8 reader keeps such a line as it is, so both readers agree on it. + let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap(); + assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}"); + assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok")); + } + + #[test] + fn dic_entries_may_hold_spaces() { + // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. + let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; + // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the + // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it + // (` lab` is no `lab`). + let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így")); + assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + assert!(!d.contains("lab") && !d.contains("labs")); + } + + #[test] + fn ignored_characters_leave_the_stripped_text_alone() { + // Hunspell removes `IGNORE` characters from the appended text, not from the stripped + // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong, + // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line). + let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap(); + assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac")); + let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap(); + assert!(add.check("ac") && add.check("acx")); + for d in [&strip, &add] { + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.check("ac"), d.check("ac")); + } + } + + #[test] + fn normalizing_borrows_when_nothing_changes() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); + assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine"))); + assert_eq!(d.normalize("(fi)ne"), "fine"); + assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot"))); + } + + #[test] + fn keepcase_stems_match_only_as_written() { + let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap(); + assert!(d.check("pH") && d.check("km") && d.check("kms")); + assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms")); + assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("km") && !back.check("Km")); + } + + #[test] + fn substandard_and_lemma_present_words_are_correct() { + // `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD + // stem is never suggested; the stems and rules they mark are still correct. + let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem")); + assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem")); + assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot")); + assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies")); + } + } + + #[test] + fn unsupported_directives_are_refused_and_the_rest_load() { + // A dictionary that needs a directive outside this engine fails to load rather than + // accepting the wrong words. + for dir in UNSUPPORTED { + assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); + } + // The directives this engine reads, with no other change to the dictionary (`AF` would + // turn the flags of `DIC` into alias numbers; it has tests of its own). + for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); + assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); + } + } + + #[test] + fn fields_split_on_spaces_and_tabs_only() { + // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator. + let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n"; + let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap(); + assert!(d.contains("koty") && d.contains("kotem")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("koty") && back.contains("kotem")); + // The same bytes as one-flag directives, as Hunspell reads them. + let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kots") && !dict.check("rossz")); + assert!(dict.check("pH") && !dict.check("PH")); + } + } + #[test] fn unsupported_and_broken_input_is_an_error() { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs new file mode 100644 index 000000000..1456f81c7 --- /dev/null +++ b/crates/proof/src/affix_suggest.rs @@ -0,0 +1,556 @@ +//! Spelling suggestions from an affix dictionary, one engine for every language that reads one +//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come +//! in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`): +//! adding a missing accent is the commonest slip and ranks first, while taking away one the +//! writer typed ranks after the known confusions; +//! 3. the REP table of common confusions (`żeka` → `rzeka`); +//! 4. a missing space (`napewno` → `na pewno`); +//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with +//! the next or mistyped (TRY letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in +//! which a MAP slip costs half (two typos at most). +//! +//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may +//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every +//! round, so a call costs a few thousand dictionary lookups at most. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters. +pub(crate) const MAX_LETTERS: usize = 40; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +pub(crate) const COST_ADD_DIACRITIC: u32 = 4; +pub(crate) const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +pub(crate) const COST_SPLIT: u32 = 12; +/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]). +const COST_SPLIT_SHORT: u32 = 17; +/// A missing space between two longer halves with no short word beside it ([`Split::Near`]). +pub(crate) const COST_SPLIT_LONG: u32 = 18; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +pub(crate) const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +/// What a language adds to its dictionary's tables. +pub(crate) struct Tuning { + /// Where a missing space may go (round 4). + pub split: Split, + /// TRY letters for a dictionary without a TRY line. + pub alphabet: &'static str, + /// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`). + pub map: &'static [&'static str], + /// REP confusions the language adds to its dictionary's. + pub rep: &'static [(&'static str, &'static str)], + /// The cost of a REP confusion. + pub rep_cost: u32, + /// Most letters a MAP variant changes. + pub max_map_changes: usize, + /// The cost of changing letter `from` to its MAP relative `to`. + pub map_cost: fn(char, char) -> u32, + /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. + pub mistype_cost: fn(&[char], usize, char) -> u32, +} + +/// Where a missing space may go. +pub(crate) enum Split { + /// Anywhere, dearer when a half is shorter than three letters. + Anywhere, + /// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one + /// that runs into the word before (`kocham cię`), dearer between two longer halves, and never + /// leaving a stray short half (`że ka`). + Near { leading: &'static [&'static str], trailing: &'static [&'static str] }, + /// The language's own rule: the cost of the halves `a b`, or `None` to skip them. + Custom(fn(&str, &str) -> Option), +} + +impl Tuning { + /// The dictionary's tables alone (Czech, Indonesian, Spanish). + pub(crate) const DEFAULT: Tuning = + Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type is likelier right, +/// so taking one away costs more than a known confusion from the REP table. +pub(crate) fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Any mistyped letter costs the same. +fn mistype_cost(_: &[char], _: usize, _: char) -> u32 { + COST_MISTYPE +} + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + /// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost). + tested: HashMap, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input { + return; + } + match self.tested.get_mut(cand) { + Some(prev) if *prev <= cost => return, + Some(prev) => *prev = cost, + None => { + self.tested.insert(cand.to_string(), cost); + } + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP groups: the dictionary's, then the language's. +fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec]> { + if t.map.is_empty() { + return Cow::Borrowed(d.map()); + } + Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect()) +} + +/// The other members of every MAP group of `c` (its likely confusions), each once. +fn alternatives(c: char, map: &[Vec]) -> Vec { + let mut out: Vec = Vec::new(); + for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) { + if x != c && !out.contains(&x) { + out.push(x); + } + } + out +} + +/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, +/// letter by letter), with its cost, each written after `buf`. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some((&c, rest)) = chars.split_first() else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); + buf.push(c); + map_variants(rest, later, t, changes, cost, buf, out); + buf.pop(); + if changes < t.max_map_changes { + for &a in here { + buf.push(a); + map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`. +pub(crate) fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered. +fn split_cost(split: &Split, a: &str, b: &str) -> Option { + let long = |part: &str| part.chars().count() >= 3; + match split { + Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }), + Split::Near { leading, trailing } => { + if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } + } + Split::Custom(cost) => cost(a, b), + } +} + +/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case. +pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + add_case(&mut col, word, &input); + let map = groups(d, t); + add_map_slips(&mut col, &chars, &map, t); + add_confusions(&mut col, &input, t); + add_splits(&mut col, &input, &t.split); + add_key_neighbours(&mut col, &chars); + add_typos(&mut col, &chars, t); + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars, &map); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let recased = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text.clone(), + }; + // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`. + let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 1: letter case. +fn add_case(col: &mut Collector, word: &str, input: &str) { + if col.d.contains(input) && word != input { + col.keep(input.to_string(), COST_CASE); + } else if col.d.contains(&capitalize(input)) && word != capitalize(input) { + col.keep(capitalize(input), COST_CASE); + } +} + +/// Round 2: accents and other MAP slips. +fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec], t: &Tuning) { + let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); + let mut variants = Vec::new(); + map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } +} + +/// Round 3: common confusions. +fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) { + let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } +} + +/// Round 4: a missing space; both halves must be dictionary words. +fn add_splits(col: &mut Collector, input: &str, split: &Split) { + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } +} + +/// Round 5: keyboard neighbours from the dictionary's KEY rows. +fn add_key_neighbours(col: &mut Collector, chars: &[char]) { + for row in col.d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } +} + +/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has +/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so +/// equal-cost candidates come out in that order. +fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) { + let d = col.d; + let n = chars.len(); + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.to_vec(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.to_vec(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.to_vec(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(chars, i, c)); + } + } + } +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { + let d = col.d; + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + /// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop) + /// ranks at the lower one. + #[test] + fn a_cheaper_later_offer_wins() { + let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); + let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + col.offer("cafe", COST_DROP_DIACRITIC); + col.offer("cafe", COST_REP); + col.offer("cafe", COST_SCAN); + assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP)); + } + + #[test] + fn distance_weighs_map_slips_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + /// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged + /// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form. + #[test] + fn keepcase_suggestions_are_not_recased_into_errors() { + let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap(); + assert!(!d.check("Km") && !d.check("KM")); + for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] { + let got = suggest_in(&d, word, 4, &Tuning::DEFAULT); + assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}"); + assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}"); + } + assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::::new()); + assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies")); + } + + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY + /// line, and a split rule decides which missing spaces are offered. + #[test] + fn tuning_adds_to_the_dictionary_tables() { + let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap(); + let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT }; + assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into())); + assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group"); + assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair"); + assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters"); + let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT }; + assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into())); + assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half"); + assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into())); + } +} diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index fee7c561c..9f8bbd0a7 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -7,88 +7,35 @@ pub(crate) mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Czech Hunspell dictionary. const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; +/// The Czech dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); + /// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Czech spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; -} - -/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. -pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() -} - -/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) + DICT.has_dictionary() } /// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals @@ -112,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Czech dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 7ab0d1493..05eba74a8 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -1,168 +1,8 @@ -//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits -//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian -//! ([`crate::id`]) uses the same ranking with its own dictionary. +//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed +//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Longest word suggestions are looked for, in letters (Spanish shares the bound). -pub(crate) const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 17; -const COST_KEY: u32 = 13; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (a capitalized name). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (typing without diacritics); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; /// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -171,237 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - let map = d.map(); - add_case_suggestions(&mut col, &input, word); - add_map_suggestions(&mut col, &chars, map); - add_replacement_suggestions(&mut col, &input, d.rep()); - add_split_suggestions(&mut col, &input); - add_keyboard_suggestions(&mut col, &chars, d.key()); - add_typing_suggestions(&mut col, &chars, d.try_chars()); - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (the first changed letter breaks ties). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -fn add_case_suggestions(col: &mut Collector, input: &str, word: &str) { - if col.d.contains(input) && word != input { - col.keep(input.to_string(), COST_CASE); - } else { - let capitalized = capitalize(input); - if col.d.contains(&capitalized) && word != capitalized { - col.keep(capitalized, COST_CASE); - } - } -} - -fn add_map_suggestions(col: &mut Collector, chars: &[char], map: &[Vec]) { - let alternatives: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(chars, &alternatives, 0, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants); - variants.sort_by_key(|variant| variant.1); - for (variant, cost) in variants { - col.offer(&variant, cost); - } -} - -fn add_replacement_suggestions(col: &mut Collector, input: &str, replacements: &[(String, String)]) { - for (from, to) in replacements { - if from.is_empty() { - continue; - } - for (index, _) in input.match_indices(from) { - if let (Some(prefix), Some(suffix)) = (input.get(..index), input.get(index + from.len()..)) { - col.offer(&format!("{prefix}{to}{suffix}"), COST_REP); - } - } - } -} - -fn add_split_suggestions(col: &mut Collector, input: &str) { - for (index, _) in input.char_indices().skip(1) { - let (Some(first), Some(second)) = (input.get(..index), input.get(index..)) else { continue }; - let cost = if first.chars().count() >= 3 && second.chars().count() >= 3 { COST_SPLIT } else { COST_SPLIT_RARE }; - col.offer(&format!("{first} {second}"), cost); - } -} - -fn add_keyboard_suggestions(col: &mut Collector, chars: &[char], key_rows: &str) { - for row in key_rows.split('|').take(32) { - let keys: Vec = row.chars().take(128).collect(); - for pair in keys.windows(2) { - let [first, second] = pair else { continue }; - for (index, &character) in chars.iter().enumerate() { - let replacement = if character == *first { - Some(*second) - } else if character == *second { - Some(*first) - } else { - None - }; - if let Some(replacement) = replacement { - let mut candidate = chars.to_vec(); - if let Some(character) = candidate.get_mut(index) { - *character = replacement; - } - col.offer(&candidate.iter().collect::(), COST_KEY); - } - } - } - } -} - -fn add_typing_suggestions(col: &mut Collector, chars: &[char], try_chars: &str) { - let mut letters: Vec = try_chars.chars().filter(|character| character.is_lowercase()).take(128).collect(); - letters.sort_unstable(); - letters.dedup(); - let word = |candidate: &[char]| candidate.iter().collect::(); - add_swap_suggestions(col, chars, &word); - add_drop_suggestions(col, chars, &word); - add_insert_suggestions(col, chars, &letters, &word); - add_mistype_suggestions(col, chars, &letters, &word); -} - -fn add_swap_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) { - for index in 0..chars.len().saturating_sub(1) { - if chars.get(index) != chars.get(index + 1) { - let mut candidate = chars.to_vec(); - candidate.swap(index, index + 1); - col.offer(&word(&candidate), COST_SWAP); - } - } -} - -fn add_drop_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) { - for index in 0..chars.len() { - let mut candidate = chars.to_vec(); - candidate.remove(index); - col.offer(&word(&candidate), COST_DROP_OR_ADD); - } -} - -fn add_insert_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) { - for index in 0..=chars.len() { - for &letter in letters { - let mut candidate = chars.to_vec(); - candidate.insert(index, letter); - col.offer(&word(&candidate), COST_DROP_OR_ADD); - } - } -} - -fn add_mistype_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) { - for index in 0..chars.len() { - for &letter in letters { - if chars.get(index) != Some(&letter) { - let mut candidate = chars.to_vec(); - if let Some(character) = candidate.get_mut(index) { - *character = letter; - } - col.offer(&word(&candidate), COST_MISTYPE); - } - } - } -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) } #[cfg(test)] mod tests { use super::*; - #[test] - fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { - let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); - let suggestions = suggest_in(&d, "počítačz", 8); - assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); - assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); - } - #[test] fn suggestions_are_fast_enough() { crate::cs::test_dict::install(); diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 351a69b04..5263bd1d9 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -16,16 +16,19 @@ mod suggest; pub(crate) mod test_dict; pub mod voseo; -use std::sync::{Arc, PoisonError, RwLock}; +use std::sync::Arc; use crate::ProofLang; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first -/// use, then the installed one (or an empty one). -static DICTS: [RwLock>>; 4] = [const { RwLock::new(None) }; 4]; +/// The dictionary of each variety, found on first use (see [`DictSlot`]). +pub(crate) static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +pub(crate) static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +pub(crate) static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +pub(crate) static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); /// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the /// generic one RLA-ES publishes). @@ -40,84 +43,38 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e /// or generic one. const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"]; -/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's -/// file names; Spain for any other language. -fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) { +/// `lang`'s variety's dictionary; Spain's for any other language. +fn slot(lang: ProofLang) -> &'static DictSlot { match lang { - ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO), - ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA), - ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY), - _ => (0, "Spanish (Spain)", SPAIN), + ProofLang::EsMx => &MEXICO_DICT, + ProofLang::Es419 => &LATIN_AMERICA_DICT, + ProofLang::EsUy => &URUGUAY_DICT, + _ => &SPAIN_DICT, } } /// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`, /// `es_UY`): the first of its file names, used for the status-bar hint and load errors. pub fn dictionary_name(lang: ProofLang) -> &'static str { - variety(lang).2.first().copied().unwrap_or("es_ES") + slot(lang).name() } /// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given /// with [`install_dictionary`], or an empty one when none is available (spelling is off then; /// see [`has_dictionary`]). pub fn dictionary(lang: ProofLang) -> Arc { - let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) }; - if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed(lang)); - let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed(lang: ProofLang) -> AffixDict { - let (_, name, files) = variety(lang); - let key = dictionary_name(lang); - let Some((aff, dic)) = crate::hunspell::find(files) else { - crate::hunspell::clear_load_error(key); - log::info!("{name} spelling: no Hunspell {key} dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("{name} spelling: {e}"); - AffixDict::default() - }) + slot(lang).get() } /// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text, /// decoded to UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - if let Some(slot) = DICTS.get(variety(lang).0) { - *slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d)); - } - Ok(()) -} - -/// Forget every variety's loaded dictionary, so the next check looks for an installed one again -/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - for slot in &DICTS { - *slot.write().unwrap_or_else(PoisonError::into_inner) = None; - } -} - -/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder -/// or the system locations now (File › Options › Proofing shows it); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path(lang: ProofLang) -> Option { - crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic) + slot(lang).install(aff, dic) } /// Is a dictionary available for `lang`'s variety (spelling is checked)? pub fn has_dictionary(lang: ProofLang) -> bool { - !dictionary(lang).is_empty() -} - -/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) + slot(lang).has_dictionary() } /// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in @@ -143,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Spanish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang)); } @@ -200,7 +145,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); for lang in VARIETIES { - let found = crate::hunspell::find_in(&dir, variety(lang).2); + let found = crate::hunspell::find_in(&dir, slot(lang).names()); assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}"); if let Some((aff, dic)) = found { assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang)); @@ -215,7 +160,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); } for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] { - assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); + assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); assert_eq!(dictionary_name(lang), name); } std::fs::remove_dir_all(dir).unwrap(); diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs index b42efa2ee..0568ae870 100644 --- a/crates/proof/src/es/suggest.rs +++ b/crates/proof/src/es/suggest.rs @@ -1,5 +1,5 @@ -//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to +//! Spanish suggestions. The ranking is the shared dictionary-driven one +//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to //! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, //! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. //! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first @@ -7,7 +7,7 @@ use crate::ProofLang; use crate::affix::{AffixDict, Casing, capitalize, casing}; -use crate::cs::suggest::MAX_LETTERS; +use crate::affix_suggest::{MAX_LETTERS, Tuning}; /// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's /// case. @@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec _ => v, }); } - for s in crate::cs::suggest::suggest_in(d, word, max) { + for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) { if !out.contains(&s) { out.push(s); } diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index ee9d58e92..53edf0674 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -9,73 +9,35 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Croatian Hunspell dictionary. const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; +/// The Croatian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); + /// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed"); - return AffixDict::default(); - }; - - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Croatian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Croatian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() -} - -/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) + DICT.has_dictionary() } /// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals @@ -99,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Croatian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index 875bf98c7..d8c19b99b 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -1,12 +1,11 @@ -//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance, -//! missing spaces and a bounded search. Proper Croatian corrections depend on -//! the actual installed dictionary entries. +//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of +//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines. -use std::collections::{HashMap, HashSet}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Short words often run into the next one (`não é`, `da água`, `para isso`). +/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`). const LEADING_WORDS: &[&str] = &[ "a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će", "bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad", @@ -14,159 +13,13 @@ const LEADING_WORDS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Croatian adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiourntsjkvlmdpbgczčćšžđfh", + map: &["cčć", "dđ", "sš", "zž"], + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -175,179 +28,24 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']]; - let map = if d.map().is_empty() { &fallback_map } else { d.map() }; - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + /// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća` + /// first, before a typo that costs more (`kuha`). + #[test] + fn diacritics_without_map_lines() { + let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap(); + assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]); + } + #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] { diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 3449c8cab..1afd903cd 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the /// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R /// and Windows-1251 the ones older Russian dictionaries use). +/// +/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes, +/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above +/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, +/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other +/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps +/// its number (the Turkish dictionary's `FLAG num`). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); - let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); + let (set, flag) = directives(aff); + let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); - Ok((text(aff, ".aff")?, text(dic, ".dic")?)) + if flag.as_deref().is_some_and(|f| f != "DEFAULT") { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); + } + Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?)) } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), @@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass +/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read, +/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8. +fn directives(aff: &[u8]) -> (Option, Option) { + let (mut set, mut flag) = (None, None); + for l in aff.split(|&b| b == b'\n') { + if set.is_some() && flag.is_some() { + break; + } + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); + let slot = match l.first() { + Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]), + Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]), + _ => continue, + }; + let arg = l + .strip_prefix(slot.1) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())); + if let Some(a) = arg { + *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase()); + } + } + (set, flag) +} + +/// Directives whose one argument is a flag. +const FLAG_DIRECTIVES: &[&str] = &[ + "CIRCUMFIX", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDFORBIDFLAG", + "COMPOUNDLAST", + "COMPOUNDFIRST", + "COMPOUNDMIDDLE", + "COMPOUNDPERMITFLAG", + "COMPOUNDROOT", + "FORBIDDENWORD", + "FORCEUCASE", + "GENERATE", + "HU_KOTOHANGZO", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "NOSUGGEST", + "ONLYINCOMPOUND", + "ONLYROOT", + "PSEUDOROOT", + "SUBSTANDARD", + "WARN", +]; + +/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian +/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. +const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; + +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what` +/// names the line in an error. +fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result { + let mut slash = None; + let mut i = 0; + while let Some(&b) = field.get(i) { + if b == b'\\' && field.get(i + 1) == Some(&b'/') { + i += 2; + continue; + } + if b == b'/' { + slash = Some(i); + break; + } + i += 1; + } + let (text, flags) = match slash { + Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), + None => (field, &[][..]), + }; + let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(); + out.push_str(&decode_latin1(flags)); + Ok(out) +} + +/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from +/// its parts is the original). +fn fields(line: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0; + let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t')); + for (i, b) in line.iter().enumerate() { + let sep = matches!(b, b' ' | b'\t'); + if sep != in_sep { + out.push(line.get(start..i).unwrap_or_default()); + start = i; + in_sep = sep; + } + } + out.push(line.get(start..).unwrap_or_default()); + out +} + +/// A UTF-8 `.aff` with single-byte flags (see [`decode`]). +fn decode_utf8_aff(aff: &[u8]) -> Result { + let mut out = String::with_capacity(aff.len()); + for (n, line) in aff.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.is_ascii() + { + out.push_str(s); + continue; + } + let parts = fields(line); + // Field index among the non-separator parts. + let words: Vec = + parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect(); + let dir = words + .first() + .and_then(|&i| parts.get(i)) + .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) + .unwrap_or_default(); + let what = || format!(".aff line {}", n + 1); + if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { + out.push_str(&String::from_utf8_lossy(line)); + continue; + } + // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or + // a flag; from there on the line is kept as text. It can only start after the directive's + // own fields, since a field such as an affix condition may itself begin with `#`. + let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); + let first_free = comment_field(&dir, field_text); + let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); + // The field index of the next non-separator part. + let mut next_field = 0; + for (i, p) in parts.iter().enumerate() { + let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| { + next_field += 1; + next_field - 1 + }); + let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); + let decoded = match (dir.as_str(), field) { + _ if in_comment => String::from_utf8_lossy(p).into_owned(), + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?, + // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` + // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. + ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()), + _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(), + }; + out.push_str(&decoded); + } + } + Ok(out) +} + +/// The first field of an `.aff` line where a trailing `# comment` may start: after the +/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its +/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field. +fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { + let number = |k: usize| field(k).is_some_and(|f| f.parse::().is_ok()); + match dir { + "PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4, + "PFX" | "SFX" => 5, + "REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN" + if number(1) && field(2).is_none_or(|f| f.starts_with('#')) => + { + 2 + } + "REP" | "ICONV" | "OCONV" | "PHONE" => 3, + "CHECKCOMPOUNDPATTERN" => 4, + _ => 2, + } +} + +/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the +/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line +/// (`kot/B 12` has the flags `B 12`, as in Hunspell). +pub(crate) fn dic_entry_end(line: &[u8]) -> usize { + let blank = |b: &u8| matches!(b, b' ' | b'\t'); + let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); + let tab = line.iter().position(|&b| b == b'\t'); + let end = match (morph, tab) { + (Some(m), Some(t)) => m.min(t), + (m, t) => match m.or(t) { + Some(e) => e, + None => line.len(), + }, + }; + // Blanks before the morphological field or the line end aren't part of the entry. + line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) +} + +/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes. +fn decode_utf8_dic(dic: &[u8]) -> Result { + let mut out = String::with_capacity(dic.len()); + for (n, line) in dic.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii()) + { + out.push_str(s); + continue; + } + let what = || format!(".dic line {}", n + 1); + // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); + // morphological fields after it are UTF-8 text. + let end = dic_entry_end(line); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?); + let rest = line.get(end..).unwrap_or_default(); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -280,6 +509,78 @@ mod tests { assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok()); } + #[test] + fn utf8_dictionaries_keep_single_byte_flags() { + // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, + // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); + let (a, d) = decode(aff, dic).unwrap(); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7")); + assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); + assert!(!dict.check("házban")); + // `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8. + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8"); + assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8"); + } + + #[test] + fn single_flag_fields_keep_their_first_byte() { + // Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the + // flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9. + let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes(); + let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap(); + assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}"); + assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}"); + assert!(d.contains("kert/\u{c3}\u{a9}")); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("kert") && dict.check("kertok")); + assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); + } + + #[test] + fn numeric_and_long_flags_stay_text() { + // `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a + // number with a non-ASCII rule or comment keeps every digit. + let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n"; + let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap(); + assert_eq!(a, aff); + assert_eq!(d, "1\nkitap/1234\n"); + let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap(); + assert!(tr.contains("kitapları") && tr.contains("kitap")); + let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n"; + assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long); + assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8"); + // `FLAG DEFAULT` is the single-byte default. + let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap(); + assert_eq!(d, "1\nkert/\u{e9}\n"); + } + + #[test] + fn flag_utf8_counts_anywhere_in_the_aff() { + // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. + let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec(); + for _ in 0..400 { + aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n"); + } + aff.extend_from_slice(b"FLAG UTF-8\r\n"); + assert!(aff.len() > 8192); + let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + } + + #[test] + fn a_comment_after_flag_utf8_keeps_unicode_flags() { + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument"); + } + #[test] fn decodes_russian_single_byte_encodings() { // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). @@ -291,6 +592,41 @@ mod tests { assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); } + #[test] + fn comments_start_after_the_directive_fields() { + let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string); + assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4); + assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`"); + assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2); + assert_eq!(comment_field("AF", f("AF Aa # 1")), 2); + assert_eq!(comment_field("REP", f("REP 12")), 2); + assert_eq!(comment_field("REP", f("REP # x")), 3); + assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4); + assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2); + // A byte flag after an append text that begins with `#` stays a flag, not a comment. + let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap(); + assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}"); + } + + #[test] + fn dic_entries_end_where_hunspell_ends_them() { + let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string(); + assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); + assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); + assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); + assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell"); + assert_eq!(end("ad hoc/B "), "ad hoc/B"); + assert_eq!(end("word"), "word"); + assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); + // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + // Without a tab or a morphological field the whole line is the entry: its byte flags + // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`). + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\n"); + } + #[test] fn libreoffice_extension_dirs_follow_the_language() { // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 012babc0e..04248d638 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -9,78 +9,41 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by /// Java locales). const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; -/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number -/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. -const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; +/// The Indonesian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); /// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Indonesian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it); `None` when there is none. A -/// dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is an Indonesian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} +/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number +/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. +const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; /// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals /// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so @@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Indonesian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w) || w.split('-').all(|p| { diff --git a/crates/proof/src/id/suggest.rs b/crates/proof/src/id/suggest.rs index 562af9a82..b42efde98 100644 --- a/crates/proof/src/id/suggest.rs +++ b/crates/proof/src/id/suggest.rs @@ -1,7 +1,6 @@ -//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem -//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is -//! hard-coded here. +//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing +//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID` +//! dictionary, so nothing Indonesian is hard-coded here. use crate::affix::AffixDict; @@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - crate::cs::suggest::suggest_in(d, word, max) + crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT) } #[cfg(test)] diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 027006a69..ee5aaaee3 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -18,72 +18,35 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Italian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Italian Hunspell dictionary (Italy first, then Switzerland). const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; +/// The Italian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); + /// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Italian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Italian spelling: no Hunspell it_IT dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Italian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is an Italian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() -} - -/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) + DICT.has_dictionary() } /// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the @@ -125,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Italian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let lower = w.to_lowercase(); if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) { return true; diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 500d1c09d..55f869091 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -1,25 +1,9 @@ -//! Italian spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`roma` → `Roma`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`, -//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`); -//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`per me`, `a Milano`, `di Roma`). const LEADING_WORDS: &[&str] = &[ @@ -35,158 +19,12 @@ const TRAILING_WORDS: &[&str] = &[ "mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove", "è", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; +/// What Italian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `à` → `a`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} /// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -195,182 +33,24 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent. - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn italian_misspellings_get_the_right_suggestion() { crate::it::test_dict::install(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 05180ea62..f92ad19a3 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -50,6 +50,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod affix; +mod affix_suggest; pub mod bg; pub mod cs; pub mod dict; @@ -65,6 +66,8 @@ pub mod patterns; pub mod pl; pub mod pt; pub mod ru; +mod slot; +mod spelling; pub mod tr; pub mod vi; @@ -157,57 +160,150 @@ pub fn ignore_word(w: &str) { } } +/// A language proofed against an installed Hunspell dictionary: all the functions below need to +/// know about it, so a language joins with one row in [`LANGUAGES`]. +struct Language { + lang: ProofLang, + slot: &'static dyn slot::Slot, + /// The spelling mark's message, in the language. + message: &'static str, + is_correct: fn(&str, ProofLang, &Options) -> bool, + suggest: fn(&str, usize, ProofLang) -> Vec, + /// Grammar issues; the flag says whether the first word must have a capital (not in a list + /// item). + grammar: fn(&str, ProofLang, bool) -> Vec, +} + +/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing +/// lists them. English has a built-in word list and is handled apart. +static LANGUAGES: [Language; 12] = [ + Language { + lang: ProofLang::Pl, + slot: &pl::DICT, + message: "Możliwy błąd pisowni", + is_correct: |w, _, o| pl::is_correct_with_options(w, o), + suggest: |w, n, _| pl::suggest(w, n), + grammar: |t, _, c| pl::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Pt, + slot: &pt::DICT, + message: "Possível erro de ortografia", + is_correct: |w, _, o| pt::is_correct_with_options(w, o), + suggest: |w, n, _| pt::suggest(w, n), + grammar: |t, _, c| pt::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::It, + slot: &it::DICT, + message: "Possibile errore di ortografia", + is_correct: |w, _, o| it::is_correct_with_options(w, o), + suggest: |w, n, _| it::suggest(w, n), + grammar: |t, _, c| it::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Hr, + slot: &hr::DICT, + message: "Moguća pravopisna pogreška", + is_correct: |w, _, o| hr::is_correct_with_options(w, o), + suggest: |w, n, _| hr::suggest(w, n), + grammar: |t, _, c| hr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Tr, + slot: &tr::DICT, + message: "Olası yazım hatası", + is_correct: |w, _, o| tr::is_correct_with_options(w, o), + suggest: |w, n, _| tr::suggest(w, n), + grammar: |t, _, c| tr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Ru, + slot: &ru::DICT, + message: "Возможна орфографическая ошибка", + is_correct: |w, _, o| ru::is_correct_with_options(w, o), + suggest: |w, n, _| ru::suggest(w, n), + grammar: |t, _, c| ru::grammar::check_with(t, c), + }, + // Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no + // exception. + Language { + lang: ProofLang::Cs, + slot: &cs::DICT, + message: "Možná pravopisná chyba", + is_correct: |w, _, o| cs::is_correct_with_options(w, o), + suggest: |w, n, _| cs::suggest(w, n), + grammar: |t, _, _| cs::grammar::check(t), + }, + Language { + lang: ProofLang::Id, + slot: &id::DICT, + message: "Kemungkinan salah eja", + is_correct: |w, _, o| id::is_correct_with_options(w, o), + suggest: |w, n, _| id::suggest(w, n), + grammar: |t, _, _| id::grammar::check(t), + }, + Language { + lang: ProofLang::Es, + slot: &es::SPAIN_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsMx, + slot: &es::MEXICO_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::Es419, + slot: &es::LATIN_AMERICA_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsUy, + slot: &es::URUGUAY_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, +]; + +/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof. +fn language(lang: ProofLang) -> Option<&'static Language> { + LANGUAGES.iter().find(|l| l.lang == lang) +} + /// Search `dir` for Hunspell dictionaries before the system locations (File › Options › /// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from /// the new search path, and text is checked again. pub fn set_dictionaries_folder(dir: Option) { hunspell::set_dictionary_dir(dir); - pl::reload_dictionary(); - pt::reload_dictionary(); - it::reload_dictionary(); - hr::reload_dictionary(); - tr::reload_dictionary(); - ru::reload_dictionary(); - cs::reload_dictionary(); - id::reload_dictionary(); - es::reload_dictionary(); + for l in &LANGUAGES { + l.slot.reload(); + } DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } /// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it /// now (`None`: not installed, so the language has no spelling marks). pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> { - vec![ - (ProofLang::Pl, pl::installed_path()), - (ProofLang::Pt, pt::installed_path()), - (ProofLang::It, it::installed_path()), - (ProofLang::Hr, hr::installed_path()), - (ProofLang::Tr, tr::installed_path()), - (ProofLang::Ru, ru::installed_path()), - (ProofLang::Cs, cs::installed_path()), - (ProofLang::Id, id::installed_path()), - (ProofLang::Es, es::installed_path(ProofLang::Es)), - (ProofLang::EsMx, es::installed_path(ProofLang::EsMx)), - (ProofLang::Es419, es::installed_path(ProofLang::Es419)), - (ProofLang::EsUy, es::installed_path(ProofLang::EsUy)), - ] + LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect() } /// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse /// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has /// checked yet reports `None` (#2830). pub fn dictionary_load_error(lang: ProofLang) -> Option { - let key = match lang { - ProofLang::Pl => "pl_PL", - ProofLang::Pt => "pt_BR", - ProofLang::It => "it_IT", - ProofLang::Hr => "hr_HR", - ProofLang::Tr => "tr_TR", - ProofLang::Id => "id_ID", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang), - _ => return None, - }; - hunspell::load_error(key).map(|(_, reason)| reason) + hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason) } /// A number that changes whenever a word is added to or removed from the user dictionary, or the @@ -438,36 +534,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p /// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English). fn dictionary_loaded(lang: ProofLang) -> Option { - Some(match lang { - ProofLang::Pl => pl::has_dictionary(), - ProofLang::Pt => pt::has_dictionary(), - ProofLang::It => it::has_dictionary(), - ProofLang::Hr => hr::has_dictionary(), - ProofLang::Tr => tr::has_dictionary(), - ProofLang::Ru => ru::has_dictionary(), - ProofLang::Cs => cs::has_dictionary(), - ProofLang::Id => id::has_dictionary(), - l if l.is_spanish() => es::has_dictionary(l), - _ => return None, - }) + language(lang).map(|l| l.slot.has_dictionary()) } /// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for; /// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`). fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> { - match lang { - _ if installed => None, - ProofLang::Pl => Some("pl_PL"), - ProofLang::Pt => Some("pt_BR"), - ProofLang::It => Some("it_IT"), - ProofLang::Hr => Some("hr_HR"), - ProofLang::Tr => Some("tr_TR"), - ProofLang::Ru => Some("ru_RU"), - ProofLang::Cs => Some("cs_CZ"), - ProofLang::Id => Some("id_ID"), - l if l.is_spanish() => Some(es::dictionary_name(l)), - _ => None, - } + language(lang).filter(|_| !installed).map(|l| l.slot.name()) } /// Is `word` spelled correctly in the run's proofing language? A language without a dictionary @@ -739,18 +812,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { return None; } - let message = match lang { - ProofLang::Pl => "Możliwy błąd pisowni", - ProofLang::Hr => "Moguća pravopisna pogreška", - ProofLang::Pt => "Possível erro de ortografia", - ProofLang::It => "Possibile errore di ortografia", - ProofLang::Tr => "Olası yazım hatası", - ProofLang::Ru => "Возможна орфографическая ошибка", - ProofLang::Cs => "Možná pravopisná chyba", - ProofLang::Id => "Kemungkinan salah eja", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico", - _ => "Possible spelling mistake", - }; + let message = language(lang).map_or("Possible spelling mistake", |l| l.message); Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) } @@ -774,40 +836,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> { /// checked as its composed form ([`composed`]). pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts), - ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts), - ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts), - ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts), - ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts), - ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word), - ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => { - accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts) - } - ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts), - ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts), - ProofLang::Bg | ProofLang::De | ProofLang::Other => true, + if lang.is_english() { + return is_correct_english(word, lang, opts); } + // A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks. + language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts)) } /// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed /// ([`composed`]) like the dictionaries' words. pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang), - ProofLang::Pl => pl::suggest(word, max), - ProofLang::Hr => hr::suggest(word, max), - ProofLang::Pt => pt::suggest(word, max), - ProofLang::It => it::suggest(word, max), - ProofLang::Tr => tr::suggest(word, max), - ProofLang::Ru => ru::suggest(word, max), - ProofLang::Cs => cs::suggest(word, max), - ProofLang::Id => id::suggest(word, max), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + if lang.is_english() { + return suggest_english(word, max, lang); } + language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang)) } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones @@ -824,18 +867,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), - ProofLang::Pl => pl::grammar::check_with(text, capital_first), - ProofLang::Hr => hr::grammar::check_with(text, capital_first), - ProofLang::Pt => pt::grammar::check_with(text, capital_first), - ProofLang::It => it::grammar::check_with(text, capital_first), - ProofLang::Tr => tr::grammar::check_with(text, capital_first), - ProofLang::Ru => ru::grammar::check_with(text, capital_first), - // Czech has no sentence-capital rule, so a list item needs no exception. - ProofLang::Cs => cs::grammar::check(text), - ProofLang::Id => id::grammar::check(text), - // Spanish has no sentence-capital rule either. - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + _ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)), } } @@ -1060,7 +1092,9 @@ mod tests { use super::*; - /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test. + /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land + /// mid-test, and by those that save the whole user dictionary and load it back, which would + /// bring back a word another test just removed. static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// #1063: the dictionaries folder is searched before the system locations; the dictionaries @@ -1077,6 +1111,7 @@ mod tests { ("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC), ("it_IT", it::test_dict::AFF, it::test_dict::DIC), ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), + ("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC), ("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC), ("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC), ("id_ID", id::test_dict::AFF, id::test_dict::DIC), @@ -1096,41 +1131,67 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it. + // Every language is listed, Russian and Czech too (#3226: they were once missing, so + // neither showed up in File › Options › Proofing). A language reports the folder's file, + // or none while a dictionary given with install_dictionary() is in use (#3252 review); + // the languages' own tests install theirs concurrently, so an exact path would race them. + // The folder is searched first whatever is in use. let found = installed_dictionaries(); - assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr)); + let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"]; assert_eq!( - found[7..], + found.iter().map(|(lang, _)| *lang).collect::>(), [ - (ProofLang::Id, Some(dir.join("id_ID.dic"))), - (ProofLang::Es, Some(dir.join("es_ES.dic"))), - (ProofLang::EsMx, Some(dir.join("es_MX.dic"))), - (ProofLang::Es419, Some(dir.join("es_419.dic"))), - (ProofLang::EsUy, Some(dir.join("es_UY.dic"))), + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy ] ); - assert_eq!( - found[..4], - [ - (ProofLang::Pl, Some(dir.join("pl_PL.dic"))), - (ProofLang::Pt, Some(dir.join("pt_BR.dic"))), - (ProofLang::It, Some(dir.join("it_IT.dic"))), - (ProofLang::Hr, Some(dir.join("hr_HR.dic"))), - ] - ); - // #3226: Russian and Czech were missing from installed_dictionaries() entirely (no - // installed_path() in either module), so neither ever showed up in File › Options › - // Proofing, installed or not. Their languages, not their exact path, is what belongs - // here: unlike the four above, installed_path() for Russian and Czech also reflects - // whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own - // test modules do, concurrently, in tests of their own — asserting an exact path here - // would race them for no reason this test needs to care about. - assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}"); + for ((lang, path), name) in found.iter().zip(names) { + let file = dir.join(format!("{name}.dic")); + assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}"); + assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}"); + } assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy)); assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); } + /// A dictionary that fails to load says why in File › Options › Proofing in every language + /// that reads one; Russian and Czech were missing from the list that named their dictionaries + /// there. + #[test] + fn every_dictionary_language_reports_its_load_error() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for (lang, _) in installed_dictionaries() { + let name = dictionary_needed(lang, false).unwrap(); + let reason = format!("zq unreadable {name}"); + let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone())); + assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}"); + hunspell::clear_load_error(name); + } + } + + /// Every proofed language other than English has a row in the language table, and only those. + #[test] + fn the_language_table_covers_the_proofed_languages() { + for l in ProofLang::PROOFED { + assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}"); + } + for (i, row) in LANGUAGES.iter().enumerate() { + assert!(row.lang.is_proofed(), "{:?}", row.lang); + assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang); + } + } + /// #1741: a list item may be a fragment, so its first word needs no capital in any language /// with the rule; a plain paragraph's does, and later sentences in the item are still checked. #[test] @@ -1593,6 +1654,7 @@ mod tests { /// #40: the user dictionary survives a save and a fresh load. #[test] fn user_dictionary_persists_through_a_file() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist")); let _ = std::fs::remove_dir_all(&dir); // The folder is created on the first save. diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 18c968feb..7cde9d0f9 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -17,78 +17,41 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Polish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Polish Hunspell dictionary. const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; +/// The Polish dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); + /// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Polish spelling: no Hunspell pl_PL dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Polish spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Polish dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced /// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`). const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"]; -/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals /// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so /// is every word when no Polish dictionary is installed. @@ -110,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Polish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let w = w.replace('’', "'"); if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs index 0e32a1eaf..7baeaa54f 100644 --- a/crates/proof/src/pl/suggest.rs +++ b/crates/proof/src/pl/suggest.rs @@ -1,26 +1,9 @@ -//! Polish spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`warszawa` → `Warszawa`); -//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy` -//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most -//! common slip, so adding them ranks first, while taking away a diacritic the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` → -//! `huśtawka`, `pszyjaciel` → `przyjaciel`); -//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham -//! cię`), cheaper when one half is a short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which a diacritic slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand -//! dictionary lookups at most. +//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`), +//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning}; /// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede /// wszystkim`). @@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[ /// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). const TRAILING_WORDS: &[&str] = &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; +/// What Polish adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx", + mistype_cost, + ..Tuning::DEFAULT +}; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moze` → `może`, then - // `Mozę`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the -/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the -/// writer did type is likelier right, so taking one away costs more than a known confusion from -/// the REP table (`żeka` → `rzeka` before `zeka`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) +/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it +/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo. +fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 { + let last = i + 1 == chars.len(); + if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE } } /// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. @@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Diacritics and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half (`że ka`). - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - // Final devoicing can make a voiced consonant sound voiceless. - let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g')); - let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }; - col.offer(&s(&v), cost); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index de4198259..ea91dbdb1 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -16,72 +16,35 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Portuguese Hunspell dictionary. const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; +/// The Portuguese dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); + /// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Portuguese spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Portuguese dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() -} - -/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) + DICT.has_dictionary() } /// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals @@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Portuguese dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs index 36f1c7b07..f8f33bff6 100644 --- a/crates/proof/src/pt/suggest.rs +++ b/crates/proof/src/pt/suggest.rs @@ -1,26 +1,9 @@ -//! Portuguese spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`brasília` → `Brasília`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`, -//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` → -//! `exceção`, `previlégio` → `privilégio`); -//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`não é`, `da água`, `para isso`). const LEADING_WORDS: &[&str] = &[ @@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[ "me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si", "eu", "tu", "que", "de", "em", "é", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Portuguese adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw", + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index c92e27125..67be70858 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -28,83 +28,35 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Russian Hunspell dictionary. const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; +/// The Russian dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); + /// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Russian spelling: no Hunspell ru_RU dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Russian spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) -} - -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; -} - -/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. -pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)? @@ -130,18 +82,30 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits or superscripts. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary (an empty one accepts everything). pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // no Russian dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'); let w = w.trim_matches('-'); let n = w.chars().count(); - if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -161,7 +125,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { return true; } if w.contains('-') { - return hyphenated(d, w); + return hyphenated(d, w, opts); } solid_compound(d, w) } @@ -340,7 +304,7 @@ fn known(d: &AffixDict, w: &str) -> bool { /// A hyphenated word that isn't in the dictionary whole: its parts, with the particles, /// prefixes and linking vowels Russian joins with a hyphen. -fn hyphenated(d: &AffixDict, w: &str) -> bool { +fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let parts: Vec<&str> = w.split('-').collect(); if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 { return false; @@ -353,7 +317,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { && lp.iter().skip(k).all(|p| PARTICLES.contains(p)) { let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default(); - return known(d, &base) || (base.contains('-') && hyphenated(d, &base)); + return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts)); } match lp.as_slice() { // `кое-кто`, `кой-какой`. @@ -369,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -497,4 +461,25 @@ mod tests { crate::add_word("Зкщфрусскоеслово"); assert!(is_correct("зкщфрусскоеслово")); } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_russian_check() { + test_dict::install(); + let lang = crate::ProofLang::Ru; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default"); + assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults)); + assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too"); + assert!(crate::is_correct_in_with("превет2", lang, &defaults)); + assert!(!crate::is_correct_in_with("превет2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index cb4c3791d..135f4152f 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -1,30 +1,14 @@ -//! Russian spelling suggestions, best first. -//! -//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the -//! common confusions are defined here. Candidates come in rounds of rising cost, each checked -//! against the dictionary: -//! 1. letter case (`москва` → `Москва`); -//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`, -//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to -//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`); -//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or -//! doubled consonant, `здраствуй` → `здравствуй`); -//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped; -//! 6. when that finds fewer than asked for: the stems that start like the word and share the -//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within -//! a letter group costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round. +//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP` +//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` → +//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule +//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning}; /// Letters confused with each other: a slip inside a group is cheaper than another typo. const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"]; - /// Common confusions: a written sequence and what was meant. const REP: &[(&str, &str)] = &[ ("жы", "жи"), @@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[ ("ц", "тс"), ("тс", "ц"), ]; - /// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`). const LEADING_WORDS: &[&str] = &[ "не", @@ -123,158 +106,42 @@ const PREPOSITIONS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"]; /// The Russian alphabet, when the dictionary has no `TRY` line. const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё"; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 3; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; +/// `е` written for `ё`, or the other way round, is barely a slip. const COST_YO: u32 = 3; -const COST_MAP: u32 = 7; -const COST_REP: u32 = 9; -const COST_SPLIT: u32 = 12; +/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`). const COST_SPLIT_PREPOSITION: u32 = 17; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} +/// What Russian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT }; -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The other members of each [`MAP`] group holding `c`. -fn alternatives(c: char) -> Vec { - let mut out: Vec = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect(); - out.sort_unstable(); - out.dedup(); - out -} - -/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip -/// (`ё` is optional in ordinary writing). +/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest. fn map_cost(from: char, to: char) -> u32 { - if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP } + if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP } } -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; +/// A missing space: cheap after a short leading word or before a short trailing one, dearer after +/// a short preposition or between two longer words, never leaving a stray short half. +fn split(a: &str, b: &str) -> Option { + let long = |p: &str| p.chars().count() >= 3; + // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for + // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. + let preposition = PREPOSITIONS.contains(&a); + if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { + return None; } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } + if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { + Some(COST_SPLIT_PREPOSITION) + } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None } } -/// A letter's group representative (the first of its first group), lower case: `ё` → `е`, -/// `з` → `з`, `с` → `з`. -fn fold(c: char) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l) -} - -/// Optimal string alignment distance where a slip within a letter group costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if fold(ai) == fold(bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} - /// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { suggest_in(&super::dictionary(), word, max) @@ -282,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Slips within a letter group. - let alts: Vec> = chars.iter().map(|&c| alternatives(c)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in REP { - for (i, _) in input.match_indices(from) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for - // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. - let preposition = PREPOSITIONS.contains(&a); - if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { - continue; - } - let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { - COST_SPLIT_PREPOSITION - } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = ALPHABET.chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a group relative of it, -/// either case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(fold).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -458,6 +161,36 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip + /// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip; + /// a preposition is never split off a verb, and after a one-letter one a typo is likelier. + #[test] + fn russian_costs_rank_its_slips() { + let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap(); + assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]); + assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]); + assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into())); + let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4); + assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}"); + assert!(s.contains(&"в доме".into()), "{s:?}"); + } + + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never + /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the + /// shared Latin-script check doesn't reach it. + #[test] + fn nosuggest_words_are_never_offered() { + let offered = |aff: &str, input: &str| { + let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap(); + suggest_in(&d, input, 16) + }; + assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared"); + for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] { + let s = offered("SET UTF-8\nNOSUGGEST N\n", input); + assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}"); + } + } + #[test] fn russian_misspellings_get_the_right_suggestion() { crate::ru::test_dict::install(); @@ -499,12 +232,15 @@ mod tests { #[test] fn distance_weighs_letter_groups_half() { + use crate::affix_suggest::{fold, weighted_distance}; + let map: Vec> = MAP.iter().map(|g| g.chars().collect()).collect(); + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); let c = |s: &str| s.chars().collect::>(); - assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1); - assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1); - assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2); - assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6); - assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3); + assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1); + assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1); + assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2); + assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6); + assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3); } #[test] diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs new file mode 100644 index 000000000..6255ae5a5 --- /dev/null +++ b/crates/proof/src/slot.rs @@ -0,0 +1,210 @@ +//! The installed Hunspell dictionary of a proofing language, in one shape for every language: +//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`]. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::affix::AffixDict; + +/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own. +pub(crate) trait Dictionary: Default + Send + Sync { + /// The installed `.aff` and `.dic` files, read. + fn load(aff: &Path, dic: &Path) -> Result; + /// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed. + fn parse(aff: &str, dic: &str) -> Result; + /// No words: the language has no spelling marks. + fn is_empty(&self) -> bool; +} + +impl Dictionary for AffixDict { + fn load(aff: &Path, dic: &Path) -> Result { + crate::hunspell::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + AffixDict::from_hunspell(aff, dic, &[]) + } + + fn is_empty(&self) -> bool { + AffixDict::is_empty(self) + } +} + +/// What every language's slot answers, whatever its dictionary type (the language table in +/// lib.rs holds them side by side). +pub(crate) trait Slot: Sync { + /// See [`DictSlot::name`]. + fn name(&self) -> &'static str; + /// See [`DictSlot::installed_path`]. + fn installed_path(&self) -> Option; + /// See [`DictSlot::has_dictionary`]. + fn has_dictionary(&self) -> bool; + /// See [`DictSlot::reload`]. + fn reload(&self); +} + +impl Slot for DictSlot { + fn name(&self) -> &'static str { + DictSlot::name(self) + } + + fn installed_path(&self) -> Option { + DictSlot::installed_path(self) + } + + fn has_dictionary(&self) -> bool { + DictSlot::has_dictionary(self) + } + + fn reload(&self) { + DictSlot::reload(self); + } +} + +/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on +/// disk (it then has no file to report, even if a matching file also happens to exist): one lock, +/// so a reload and an install can't interleave their two fields. +struct State { + dict: Option>, + in_memory: bool, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, +} + +/// One language's dictionary. +pub(crate) struct DictSlot { + /// File names the dictionary may have, the usual one first (also the key of its load error). + names: &'static [&'static str], + /// The language's English name, for the log. + english: &'static str, + state: RwLock>, +} + +impl DictSlot { + pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) } + } + + fn read(&self) -> RwLockReadGuard<'_, State> { + self.state.read().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self) -> RwLockWriteGuard<'_, State> { + self.state.write().unwrap_or_else(PoisonError::into_inner) + } + + /// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names. + pub(crate) fn name(&self) -> &'static str { + self.names.first().copied().unwrap_or_default() + } + + /// The file names the dictionary may have, the usual one first. + #[cfg(test)] + pub(crate) fn names(&self) -> &'static [&'static str] { + self.names + } + + /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or + /// an empty one when none is available (the language's spelling is off then). + pub(crate) fn get(&self) -> Arc { + let reloads = { + let s = self.read(); + if let Some(d) = s.dict.as_ref() { + return d.clone(); + } + s.reloads + }; + self.keep(reloads, Arc::new(self.load_installed())) + } + + /// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the + /// load may have read the old folder: it serves this one check and the next one loads again. + fn keep(&self, reloads: u64, loaded: Arc) -> Arc { + let mut s = self.write(); + if s.reloads != reloads { + return s.dict.clone().unwrap_or(loaded); + } + // Another thread (or an install) may have filled it meanwhile: that one wins. + s.dict.get_or_insert(loaded).clone() + } + + fn load_installed(&self) -> D { + let name = self.name(); + let Some((aff, dic)) = crate::hunspell::find(self.names) else { + crate::hunspell::clear_load_error(name); + log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english); + return D::default(); + }; + crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("{} spelling: {e}", self.english); + D::default() + }) + } + + /// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for + /// example one the user picked. On error the current dictionary stays. + pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> { + let d = D::parse(aff, dic)?; + let mut slot = self.write(); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; + Ok(()) + } + + /// Forget the dictionary, so the next check looks for an installed one again (after + /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). + pub(crate) fn reload(&self) { + let mut slot = self.write(); + slot.dict = None; + slot.in_memory = false; + slot.reloads = slot.reloads.wrapping_add(1); + } + + /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the + /// system locations now (File › Options › Proofing shows it); `None` when there is none, or + /// when the dictionary in use was given with [`Self::install`] and so has no file. + pub(crate) fn installed_path(&self) -> Option { + if self.read().in_memory { + return None; + } + crate::hunspell::find(self.names).map(|(_, dic)| dic) + } + + /// Is a dictionary available (spelling is checked)? + pub(crate) fn has_dictionary(&self) -> bool { + !self.get().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static SLOT: DictSlot = DictSlot::new(&["zz_SLOT_TEST"], "Test"); + + #[test] + fn an_installed_dictionary_has_no_file_until_reloaded() { + assert!(!SLOT.has_dictionary(), "nothing installed under this name"); + assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(!SLOT.has_dictionary(), "a refused install keeps the current one"); + SLOT.install("", "1\nkot\n").unwrap(); + assert!(SLOT.has_dictionary() && SLOT.get().check("kot")); + assert_eq!(SLOT.installed_path(), None); + SLOT.reload(); + assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); + assert_eq!(SLOT.name(), "zz_SLOT_TEST"); + } + + /// A load that started before a dictionaries-folder change is not kept after its reload. + #[test] + fn a_load_from_before_a_reload_is_not_kept() { + let slot: DictSlot = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test"); + let before = slot.read().reloads; + slot.reload(); + let _ = slot.keep(before, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_none(), "the next check loads from the new folder"); + let now = slot.read().reloads; + let _ = slot.keep(now, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_some(), "a load since the reload is kept"); + } +} diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs new file mode 100644 index 000000000..ab5e520e9 --- /dev/null +++ b/crates/proof/src/spelling.rs @@ -0,0 +1,52 @@ +//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian, +//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the +//! File › Options › Proofing choices. Each language goes on from there with its own rules. + +use crate::Options; +use crate::affix::AffixDict; + +/// Is `c` a letter of the Latin script (the only one these dictionaries cover)? +fn latin(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it +/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single +/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the +/// options skip those, or it is in another script. +pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> { + if d.is_empty() { + return None; + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + return None; + } + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return None; + } + w.chars().any(latin).then_some(w) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn words_needing_no_lookup() { + let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); + let defaults = Options::default(); + assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary"); + for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] { + assert_eq!(word_to_check(&d, w, &defaults), None, "{w}"); + } + assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot")); + assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot")); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO")); + assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B")); + assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms"); + } +} diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index 4b59ccc29..8cbacf02e 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -26,6 +26,20 @@ pub struct TrDict { max_suffix_len: usize, } +impl crate::slot::Dictionary for TrDict { + fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + super::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + TrDict::from_hunspell(aff, dic) + } + + fn is_empty(&self) -> bool { + TrDict::is_empty(self) + } +} + impl TrDict { /// Returns true if empty (no dictionary loaded). pub fn is_empty(&self) -> bool { diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index 25ee424c4..967148d8b 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -15,39 +15,26 @@ pub mod dict; pub mod grammar; pub mod hyphen; pub mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; + +use crate::slot::DictSlot; pub use dict::TrDict; pub use suggest::suggest; -/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Turkish Hunspell dictionary. const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; +/// The Turkish dictionary, found on first use (see [`DictSlot`]). +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); + /// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available. pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> TrDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); - return TrDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Turkish spelling: {e}"); - TrDict::default() - }) + DICT.get() } /// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line, @@ -57,28 +44,15 @@ pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result Result<(), String> { - let d = TrDict::from_hunspell(aff, dic)?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) -} - -/// Reload the dictionary on next check. -pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; -} - -/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. -pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.install(aff, dic) } /// Is a Turkish dictionary available? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `word` spelled correctly in Turkish? @@ -86,15 +60,26 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain +/// numbers off, words with digits are checked against the dictionary too. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary. pub fn is_correct_with(d: &TrDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // No Turkish dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); let w = w.trim_matches(['\'', '’']); let n = w.chars().count(); - if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return true; } // Check user dictionary first @@ -119,7 +104,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool { } // Compound words separated by hyphens (e.g., `afyon-karahisar`) - if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) { + if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) { return true; } @@ -145,11 +130,8 @@ mod tests { } #[test] - fn installed_dictionary_checks_words_if_present() { - // Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check. - if !has_dictionary() { - return; - } + fn installed_dictionary_checks_words() { + test_dict::install(); assert!(is_correct("kitap")); assert!(is_correct("kitaplar")); assert!(is_correct("evde")); @@ -164,4 +146,19 @@ mod tests { crate::add_word("benimbilmedigimkelime"); assert!(is_correct("benimbilmedigimkelime")); } + + /// With Ignore words that contain numbers off, Turkish checks words with digits against the + /// dictionary instead of accepting them, hyphen parts included. + #[test] + fn proofing_options_reach_the_turkish_check() { + test_dict::install(); + let lang = crate::ProofLang::Tr; + let defaults = crate::Options::default(); + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("kitapp2", lang, &defaults)); + assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("kitapp2"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index c5f4db895..08874420e 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -318,10 +318,7 @@ mod tests { #[test] fn typo_transposition_is_suggested() { - // Suggestions come from the installed Turkish dictionary; without one there are none. - if !super::super::has_dictionary() { - return; - } + crate::tr::test_dict::install(); let sugs = suggest("sorna", 3); assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}"); diff --git a/crates/proof/src/tr/test_dict.rs b/crates/proof/src/tr/test_dict.rs new file mode 100644 index 000000000..de4a13195 --- /dev/null +++ b/crates/proof/src/tr/test_dict.rs @@ -0,0 +1,22 @@ +//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// No affix rules: the forms the tests look up are listed whole. +pub const AFF: &str = "SET UTF-8\n"; + +/// The words the Turkish tests expect to be known. +pub const DIC: &str = "7 +kitap +kitaplar +ev +evde +olacaklardan +sonra +geldin +"; + +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +}