Merge pull request #5000 from benandez/integration/batch-proof

Integration batch proof: 2 approved proofing PRs (merge commit, please)
This commit is contained in:
aiongg
2026-11-01 10:46:45 -05:00
committed by GitHub
27 changed files with 2255 additions and 2902 deletions
+25 -5
View File
@@ -8472,9 +8472,18 @@ fn next_footnote_skips_endnotes() {
assert_eq!(s.sel.focus.off, 9);
}
/// Tests that install a test dictionary wait for the dictionaries-folder test, which changes the
/// folder and then reads what each language reports: an install in between reports no file.
static DICTIONARIES: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn dictionaries_turn() -> std::sync::MutexGuard<'static, ()> {
DICTIONARIES.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
/// A tiny synthetic Polish Hunspell dictionary (written for these tests). Real dictionaries are
/// installed on the system, never bundled; every Polish test installs this same one.
fn polish_test_dictionary() {
let _turn = dictionaries_turn();
wordcraft_proof::pl::install_dictionary(POLISH_TEST_AFF, POLISH_TEST_DIC).unwrap();
}
@@ -8484,6 +8493,7 @@ const POLISH_TEST_DIC: &str = "12\npłynie\nrzeka\na\nto\nwiem\nże\ntak\nw\nog
/// A tiny synthetic Portuguese Hunspell dictionary (written for these tests). Real dictionaries
/// are installed on the system, never bundled; every Portuguese test installs this same one.
fn portuguese_test_dictionary() {
let _turn = dictionaries_turn();
wordcraft_proof::pt::install_dictionary(PORTUGUESE_TEST_AFF, PORTUGUESE_TEST_DIC).unwrap();
}
@@ -8497,6 +8507,7 @@ const PORTUGUESE_TEST_DIC: &str = "11\nsaudade\ncoração\ne\nele\ndisse\nnão\n
/// running meanwhile see the same words; it stays set for the rest of the run for that reason.
#[test]
fn dictionaries_folder_option_is_saved_searched_and_reported() {
let _turn = dictionaries_turn();
let dir = std::env::temp_dir().join(format!("wc-engine-dictionaries-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
for (name, aff, dic) in [
@@ -8527,14 +8538,20 @@ fn dictionaries_folder_option_is_saved_searched_and_reported() {
assert_eq!(r["dictionariesFolder"], json!(dir.to_string_lossy()));
let dicts = r["dictionaries"].as_array().unwrap();
let file = |lang: &str| dicts.iter().find(|d| d["language"] == lang).map(|d| d["file"].clone());
assert_eq!(file("pl"), Some(json!(dir.join("pl_PL.dic").to_string_lossy())));
assert_eq!(file("pt"), Some(json!(dir.join("pt_BR.dic").to_string_lossy())));
assert_eq!(file("id"), Some(json!(dir.join("id_ID.dic").to_string_lossy())));
// Each language reports the folder's file, which is searched first.
let reported = |got: Option<&serde_json::Value>, name: &str| {
let want = dir.join(format!("{name}.dic"));
assert_eq!(got, Some(&json!(want.to_string_lossy())), "{name}: {dicts:?}");
assert_eq!(wordcraft_proof::hunspell::find(&[name]).map(|(_, dic)| dic), Some(want));
};
reported(file("pl").as_ref(), "pl_PL");
reported(file("pt").as_ref(), "pt_BR");
reported(file("id").as_ref(), "id_ID");
// Each Spanish variety reports its own file (Spain, Mexico, Latin America, Uruguay, in order).
let spanish: Vec<serde_json::Value> = dicts.iter().filter(|d| d["language"] == "es").map(|d| d["file"].clone()).collect();
assert_eq!(spanish.len(), 4);
assert_eq!(spanish.first(), Some(&json!(dir.join("es_ES.dic").to_string_lossy())));
assert_eq!(spanish.last(), Some(&json!(dir.join("es_UY.dic").to_string_lossy())));
reported(spanish.first(), "es_ES");
reported(spanish.last(), "es_UY");
// Polish text is checked against the folder's dictionary.
run(&mut s, "document.setText", json!({"text": "rzeka płynie żeka"}));
run(&mut s, "select.all", json!({}));
@@ -8584,6 +8601,7 @@ fn portuguese_proofing_through_the_review_commands() {
}
fn indonesian_test_dictionary() {
let _turn = dictionaries_turn();
wordcraft_proof::id::install_dictionary(INDONESIAN_TEST_AFF, INDONESIAN_TEST_DIC).unwrap();
}
@@ -8755,6 +8773,7 @@ const URUGUAYAN_TEST_DIC: &str = "10\nesto\nes\nuna\nprueba/S\ndel\ncorrector\nt
fn spanish_test_dictionaries() {
use wordcraft_proof::ProofLang;
let _turn = dictionaries_turn();
for (lang, dic) in [(ProofLang::Es, SPANISH_TEST_DIC), (ProofLang::EsUy, URUGUAYAN_TEST_DIC)] {
wordcraft_proof::es::install_dictionary(lang, SPANISH_TEST_AFF, dic).unwrap();
}
@@ -8883,6 +8902,7 @@ const ITALIAN_TEST_AFF: &str = "SET UTF-8\nTRY aeiortnsclcaffèéè\nMAP 2\nMAP
const ITALIAN_TEST_DIC: &str = "6\ncasa\ncaffè\nroma\ndisse\ne\nper\n";
fn italian_test_dictionary() {
let _turn = dictionaries_turn();
wordcraft_proof::it::install_dictionary(ITALIAN_TEST_AFF, ITALIAN_TEST_DIC).unwrap();
}
+501 -109
View File
@@ -11,10 +11,14 @@
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
//!
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`),
//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and
//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes
//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the
//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi`
//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP`
//! and `REP`.
//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding,
//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct.
//!
//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and
@@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX;
/// Directives that change which words are correct in ways this engine doesn't implement.
const UNSUPPORTED: &[&str] = &[
"AF",
"AM",
"CHECKSHARPS",
"CIRCUMFIX",
"COMPLEXPREFIXES",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFIRST",
"COMPOUNDFLAG",
"COMPOUNDLAST",
"COMPOUNDMIDDLE",
"COMPOUNDMIN",
"COMPOUNDPERMITFLAG",
"COMPOUNDRULE",
"FULLSTRIP",
"ICONV",
"IGNORE",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"OCONV",
"ONLYINCOMPOUND",
"PSEUDOROOT",
"SUBSTANDARD",
];
/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary
/// uses 158).
const MAX_FLAGS: usize = 256;
/// A set of flags, one bit per distinct flag character of the dictionary.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
struct Mask([u64; MAX_FLAGS / 64]);
impl Mask {
const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]);
/// The set holding flag number `i` alone (empty when `i` is out of range).
fn bit(i: usize) -> Mask {
let mut m = Mask::EMPTY;
if let Some(w) = m.0.get_mut(i / 64) {
*w = 1 << (i % 64);
}
m
}
/// The number of the lowest flag in the set, if any.
fn first(self) -> Option<usize> {
self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize)
}
fn is_empty(self) -> bool {
self.0.iter().all(|&w| w == 0)
}
/// Do the two sets share a flag?
fn meets(self, other: Mask) -> bool {
!(self & other).is_empty()
}
/// Does this set hold every flag of `other`?
fn covers(self, other: Mask) -> bool {
(self & other) == other
}
}
impl std::ops::BitAnd for Mask {
type Output = Mask;
fn bitand(mut self, o: Mask) -> Mask {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a &= b;
}
self
}
}
impl std::ops::BitOr for Mask {
type Output = Mask;
fn bitor(mut self, o: Mask) -> Mask {
self |= o;
self
}
}
impl std::ops::BitOrAssign for Mask {
fn bitor_assign(&mut self, o: Mask) {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a |= b;
}
}
}
/// One character class of a rule's condition.
#[derive(Clone, Debug, PartialEq, Eq)]
enum Cond {
@@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18;
#[derive(Clone, Debug)]
struct Rule {
/// Bit of the rule's flag in the stems' flag masks.
bit: u128,
bit: Mask,
flag: char,
cross: bool,
strip: Box<str>,
add: Box<str>,
/// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing
/// further affixes. Chained application is bounded by [`MAX_CHAIN`].
cont: u128,
cont: Mask,
cond: Box<[Cond]>,
cond_text: Box<str>,
}
@@ -174,11 +239,13 @@ pub struct AffixDict {
/// Per stem: index into `sets`.
flags: Vec<u16>,
/// Distinct flag masks.
sets: Vec<u128>,
sets: Vec<Mask>,
/// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared).
forbidden: u128,
forbidden: Mask,
/// Stems with NOSUGGEST are correct, but none of their forms are suggested.
no_suggest: u128,
no_suggest: Mask,
/// Stems with SUBSTANDARD are correct too, and likewise never suggested.
substandard: Mask,
/// Open-addressing hash table of stem indices (`EMPTY` = free).
table: Vec<u32>,
/// Flag char of each bit.
@@ -195,6 +262,12 @@ pub struct AffixDict {
key: String,
map: Vec<Vec<char>>,
rep: Vec<(String, String)>,
/// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none).
keepcase: Mask,
/// `ICONV`: input conversions applied before a lookup, `(from, to)`.
iconv: Vec<(String, String)>,
/// `IGNORE`: characters removed from input words (and from the stems and appended texts).
ignore: Box<[char]>,
/// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
comments: Vec<String>,
}
@@ -263,6 +336,57 @@ struct Rules {
/// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct.
forbidden: Option<char>,
no_suggest: Option<char>,
substandard: Option<char>,
/// The `KEEPCASE` flag.
keepcase: Option<char>,
iconv: Vec<(String, String)>,
ignore: Vec<char>,
/// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.
aliases: Vec<String>,
/// The `AF` table opened with its count line, so flag fields are alias numbers. A table
/// without one is rejected, as in Hunspell: its lines are ignored.
aliases_on: bool,
}
impl Rules {
/// Flags as written in a `.dic` entry or a continuation class: the flag characters
/// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads
/// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags:
/// the stem is still a word, the rule still applies, without the flags (Hunspell warns).
fn flags<'a>(&'a self, written: &'a str) -> &'a str {
if !self.aliases_on {
return written;
}
let digits = written.bytes().take_while(u8::is_ascii_digit).count();
written
.get(..digits)
.and_then(|d| d.parse::<usize>().ok())
.and_then(|n| n.checked_sub(1))
.and_then(|i| self.aliases.get(i))
.map_or("", String::as_str)
}
/// `w` without the `IGNORE` characters.
fn strip_ignored(&self, w: &str) -> String {
if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }
}
}
/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a
/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag.
fn split_fields(line: &str) -> impl Iterator<Item = &str> {
line.split([' ', '\t']).filter(|f| !f.is_empty())
}
/// The one flag character a directive (`FORBIDDENWORD w`) names.
fn one_flag(arg: &str, dir: &str, n: usize) -> Result<char, String> {
let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
// Only the field separators of `split_fields` can't be a flag: a flag byte read as
// U+0085 or U+00A0 is one.
(Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c),
_ => Err(format!("line {}: {dir} needs one flag character", n + 1)),
}
}
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
@@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
let mut cross: HashMap<(bool, char), bool> = HashMap::new();
let mut map_header = false;
let mut rep_header = false;
let mut af_header = false;
let mut iconv_header = false;
for (n, line) in aff.lines().enumerate() {
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let Some(&dir) = f.first() else { continue };
if dir.starts_with('#') {
continue;
@@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
match dir {
"SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS"
| "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}
// Morphological aliases, and a flag that only matters for morphological generation:
// the words it marks are correct.
"AM" | "LEMMA_PRESENT" => {}
"SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?),
// The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
// trailing comment (`AF 1559 # aliases`).
"AF" => {
if !af_header {
af_header = true;
// A count below 1 rejects the table too: flags stay literal (`AF 0`).
r.aliases_on = at(1).parse::<usize>().is_ok_and(|n| n > 0);
} else if r.aliases_on {
r.aliases.push(at(1).to_string());
}
}
"ICONV" => {
if !iconv_header && at(1).parse::<usize>().is_ok() {
iconv_header = true;
} else if f.len() >= 3 {
r.iconv.push((at(1).to_string(), at(2).to_string()));
}
}
"IGNORE" => r.ignore = at(1).chars().collect(),
"KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?),
// Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
// with the one-character parsing here; only the multi-character encodings differ.
"FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => {
return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
}
"FLAG" => {}
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
r.forbidden = Some(c);
} else {
r.no_suggest = Some(c);
}
}
_ => return Err(format!("line {}: {directive} needs one flag character", n + 1)),
}
}
"FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?),
"NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?),
"TRY" => r.try_chars = cap_try(at(1)),
"KEY" => r.key = at(1).to_string(),
"MAP" => {
@@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
}
}
"REP" => {
if !rep_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
if !rep_header && at(1).parse::<usize>().is_ok() {
rep_header = true;
} else if f.len() >= 3 {
r.rep.push((at(1).replace('_', " "), at(2).replace('_', " ")));
@@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
_ => {}
}
}
// Aliases may be declared after the rules that use them; resolve once all are read.
// Hunspell removes `IGNORE` characters from the appended text only: a stripped text that
// holds one never matches a stem (stems lose them too), so the rule never applies.
let resolve = |rules: &mut Vec<RawRule>, r: &Rules| {
for rule in rules.iter_mut() {
rule.4 = r.flags(&rule.4).to_string();
rule.3 = r.strip_ignored(&rule.3);
}
};
let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));
resolve(&mut pfx, &r);
resolve(&mut sfx, &r);
(r.prefixes, r.suffixes) = (pfx, sfx);
Ok(r)
}
@@ -397,7 +549,12 @@ impl AffixDict {
let rules = parse_aff(aff)?;
let mut stems: Vec<(String, String)> = Vec::new();
for (n, line) in dic.lines().enumerate() {
let Some(tok) = line.split_whitespace().next() else { continue };
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
// Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).
let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();
if tok.is_empty() {
continue;
}
if n == 0 && tok.parse::<usize>().is_ok() {
continue;
}
@@ -420,9 +577,11 @@ impl AffixDict {
word.push(c);
}
}
let word = rules.strip_ignored(&word);
if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
continue;
}
let flags = rules.flags(&flags).to_string();
stems.push((word, flags));
}
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
@@ -434,11 +593,12 @@ impl AffixDict {
flag_chars.extend(stems.iter().flat_map(|s| s.1.chars()));
flag_chars.sort_unstable();
flag_chars.dedup();
if flag_chars.len() > 128 {
return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len()));
if flag_chars.len() > MAX_FLAGS {
return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len()));
}
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i);
let forbidden = rules.forbidden.map_or(0, bit_of);
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit);
let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of);
let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of);
// Stems sorted by bytes; homonyms (the same word twice) are merged.
stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
let mut d = AffixDict {
@@ -446,7 +606,11 @@ impl AffixDict {
offs: Vec::with_capacity(stems.len() + 1),
flag_chars: flag_chars.clone(),
forbidden,
no_suggest: rules.no_suggest.map_or(0, bit_of),
no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of),
substandard: rules.substandard.map_or(Mask::EMPTY, bit_of),
keepcase,
iconv: rules.iconv,
ignore: rules.ignore.into_boxed_slice(),
try_chars: rules.try_chars,
key: rules.key,
map: rules.map,
@@ -455,16 +619,16 @@ impl AffixDict {
..AffixDict::default()
};
d.offs.push(0);
let mut set_ids: HashMap<u128, u16> = HashMap::new();
let mut set_ids: HashMap<Mask, u16> = HashMap::new();
let mut i = 0;
while i < stems.len() {
let Some((word, _)) = stems.get(i) else { break };
let mut mask = 0u128;
let mut mask = Mask::EMPTY;
let mut j = i;
while let Some((w, f)) = stems.get(j)
&& w == word
{
mask |= f.chars().map(bit_of).fold(0, |a, b| a | b);
mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b);
j += 1;
}
let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?;
@@ -478,7 +642,7 @@ impl AffixDict {
i = j;
}
let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result<Rule, String> {
let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c));
let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c));
Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() })
};
d.prefixes = rules.prefixes.into_iter().map(mk).collect::<Result<_, _>>()?;
@@ -493,7 +657,7 @@ impl AffixDict {
// Each (appended, stripped) pair's group, found by hash rather than by scanning the
// appended string's groups: a dictionary with many strips per suffix stays linear.
let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); MAX_FLAGS];
for (i, r) in self.suffixes.iter().enumerate() {
let groups = idx.entry(r.add.clone()).or_default();
let Ok(i) = u32::try_from(i) else { break };
@@ -504,9 +668,7 @@ impl AffixDict {
groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
}
}
if r.bit != 0
&& let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
{
if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) {
list.push(i);
}
}
@@ -551,12 +713,12 @@ impl AffixDict {
}
/// The flag characters of `mask`, in flag order (how the compact format stores flags).
fn flag_chars_for(&self, mask: u128) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect()
fn flag_chars_for(&self, mask: Mask) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect()
}
/// The character of `mask` when it holds exactly one flag.
fn flag_char(&self, mask: u128) -> Option<char> {
fn flag_char(&self, mask: Mask) -> Option<char> {
let s = self.flag_chars_for(mask);
let mut it = s.chars();
match (it.next(), it.next()) {
@@ -589,11 +751,12 @@ impl AffixDict {
}
/// Flag mask of a stem, if `w` is one.
fn stem_flags(&self, w: &str) -> Option<u128> {
self.stem_flags_excluding(w, self.forbidden)
fn stem_flags(&self, w: &str) -> Option<Mask> {
self.stem_flags_except(w, Mask::EMPTY)
}
fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option<u128> {
/// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`.
fn stem_flags_except(&self, w: &str, deny: Mask) -> Option<Mask> {
if self.table.is_empty() {
return None;
}
@@ -607,8 +770,9 @@ impl AffixDict {
if self.word(t as usize) == Some(w) {
let id = *self.flags.get(t as usize)?;
let mask = self.sets.get(usize::from(id)).copied();
// All derivations use this lookup: exclusions apply to stems and their forms.
return mask.filter(|m| m & excluded == 0);
// A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit,
// and (this is the only stem lookup) no affixed form is built on it either.
return mask.filter(|m| !m.meets(self.forbidden | deny));
}
slot = (slot + 1) & mask;
}
@@ -621,8 +785,8 @@ impl AffixDict {
/// The rules of a group that could lead to `w` are merged into one mask of their flags and
/// followed once, not once per rule: the answer is the same, but thousands of rules of one
/// flag cost one walk, not one each (#3613).
fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool {
if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) {
return true;
}
if depth == 0 {
@@ -641,9 +805,9 @@ impl AffixDict {
.rules
.iter()
.filter_map(|&i| self.suffixes.get(i as usize))
.filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
.fold(0u128, |m, r| m | r.bit);
if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
.filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true))
.fold(Mask::EMPTY, |m, r| m | r.bit);
if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) {
return true;
}
}
@@ -651,21 +815,55 @@ impl AffixDict {
false
}
/// Is `w` exactly (case and all) a stem or an affixed form of one?
/// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE`
/// apply first.
pub fn contains(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden)
self.contains_except(&self.normalize(w), Mask::EMPTY)
}
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded).
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD
/// stems excluded).
pub fn contains_suggestible(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden | self.no_suggest)
self.contains_except(&self.normalize(w), self.no_suggest | self.substandard)
}
fn contains_excluding(&self, w: &str, excluded: u128) -> bool {
/// `w` after the `ICONV` conversions (at each position the longest match wins) and without
/// the `IGNORE` characters.
fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
if self.iconv.is_empty() && self.ignore.is_empty() {
return std::borrow::Cow::Borrowed(w);
}
// One pass: the input is copied only from the first change on.
let mut out: Option<String> = None;
let mut rest = w;
while let Some(c) = rest.chars().next() {
let done = w.get(..w.len() - rest.len()).unwrap_or_default();
let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
match hit {
Some((from, to)) => {
out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));
rest = rest.get(from.len()..).unwrap_or_default();
}
None => {
if self.ignore.contains(&c) {
out.get_or_insert_with(|| done.to_string());
} else if let Some(o) = out.as_mut() {
o.push(c);
}
rest = rest.get(c.len_utf8()..).unwrap_or_default();
}
}
}
out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
}
/// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't
/// count (case variants of `KEEPCASE` stems).
fn contains_except(&self, w: &str, deny: Mask) -> bool {
if w.is_empty() || w.len() > MAX_WORD_BYTES {
return false;
}
if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) {
if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) {
return true;
}
let mut root = String::with_capacity(w.len() + 8);
@@ -680,13 +878,13 @@ impl AffixDict {
if !r.cond_matches(&root, false) {
continue;
}
if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) {
if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) {
return true;
}
if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) {
if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) {
return true;
}
if r.cross && self.suffixed(&root, r.bit, excluded) {
if r.cross && self.suffixed(&root, r.bit, deny) {
return true;
}
}
@@ -695,7 +893,7 @@ impl AffixDict {
/// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must
/// carry those flags too and the suffix must allow cross products.
fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool {
fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool {
let mut root = String::with_capacity(w.len() + 8);
// Each split point from the end: the appended string is `w[b..]`, at most the longest
// suffix, and something must stay before it.
@@ -710,18 +908,18 @@ impl AffixDict {
let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize));
// The stem path: `root` is a stem carrying the rule's flag (and the prefix's,
// when a cross product is being checked).
if let Some(f) = self.stem_flags_excluding(&root, excluded)
&& f & need == need
&& rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true))
if let Some(f) = self.stem_flags_except(&root, deny)
&& f.covers(need)
&& rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true))
{
return true;
}
// Continuation classes: `root` need not be a stem carrying this rule's flag —
// it may be a form that gained the flag from another rule's continuation. The
// matching rules' flags are looked for together, in one walk.
if need == 0 {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
if need.is_empty() {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit);
if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) {
return true;
}
}
@@ -732,26 +930,35 @@ impl AffixDict {
/// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals
/// — in lower case (and capitalized), since a sentence may start with any word and headings
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`).
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a
/// `KEEPCASE` stem matches only as written.
pub fn check(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden)
self.check_except(w, Mask::EMPTY)
}
/// Whether a suggestion is correct in its displayed case and has an eligible stem.
pub fn can_suggest(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden | self.no_suggest)
self.check_except(w, self.no_suggest | self.substandard)
}
fn check_excluding(&self, w: &str, excluded: u128) -> bool {
if self.contains_excluding(w, excluded) {
/// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case
/// alone (`km` from `km/K`, which `Km` and `KM` are not).
pub(crate) fn case_locked(&self, w: &str) -> bool {
self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase)
}
fn check_except(&self, w: &str, deny: Mask) -> bool {
let w = self.normalize(w);
if self.contains_except(&w, deny) {
return true;
}
match casing(w) {
let deny = deny | self.keepcase;
match casing(&w) {
Casing::Lower | Casing::Mixed => false,
Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded),
Casing::Initial => self.contains_except(&w.to_lowercase(), deny),
Casing::Upper => {
let lower = w.to_lowercase();
self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded)
self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny)
}
}
}
@@ -766,11 +973,11 @@ impl AffixDict {
let mut cross_sfx = Vec::new();
// Suffix forms queued with the continuation classes they carry, for chained expansion;
// each (form, classes) pair is expanded once.
let mut chain: Vec<(String, u128)> = Vec::new();
let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
let mut chain: Vec<(String, Mask)> = Vec::new();
let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new();
let mut work = 0usize;
for r in &self.suffixes {
if f & r.bit == 0 || !r.cond_matches(stem, true) {
if !f.meets(r.bit) || !r.cond_matches(stem, true) {
continue;
}
let Some(base) = stem.strip_suffix(&*r.strip) else { continue };
@@ -778,7 +985,7 @@ impl AffixDict {
if r.cross {
cross_sfx.push(form.clone());
}
if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) {
chain.push((form.clone(), r.cont));
}
out.push(form);
@@ -804,7 +1011,7 @@ impl AffixDict {
}
let Some(base) = form.strip_suffix(&*r.strip) else { continue };
let next = format!("{base}{}", r.add);
if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) {
chain.push((next.clone(), r.cont));
}
out.push(next);
@@ -815,7 +1022,7 @@ impl AffixDict {
}
}
for r in &self.prefixes {
if f & r.bit == 0 || !r.cond_matches(stem, false) {
if !f.meets(r.bit) || !r.cond_matches(stem, false) {
continue;
}
let Some(rest) = stem.strip_prefix(&*r.strip) else { continue };
@@ -839,11 +1046,11 @@ impl AffixDict {
}
/// The suffix rules carrying any flag of `mask`, in rule order.
fn suffixes_with(&self, mask: u128) -> Vec<u32> {
fn suffixes_with(&self, mask: Mask) -> Vec<u32> {
let mut rules: Vec<u32> = Vec::new();
let mut lists = 0;
for (bit, list) in self.sfx_by_flag.iter().enumerate() {
if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
if mask.meets(Mask::bit(bit)) && !list.is_empty() {
rules.extend_from_slice(list);
lists += 1;
}
@@ -904,13 +1111,21 @@ impl AffixDict {
for (a, b) in &self.rep {
out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_")));
}
for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] {
if mask != 0
for (directive, mask) in
[("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)]
{
if !mask.is_empty()
&& let Some(c) = self.flag_char(mask)
{
out.push_str(&format!("{directive} {c}\n"));
}
}
for (a, b) in &self.iconv {
out.push_str(&format!("ICONV {a} {b}\n"));
}
if !self.ignore.is_empty() {
out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));
}
let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() };
for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] {
for r in rules {
@@ -934,8 +1149,8 @@ impl AffixDict {
let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255);
raw.push(p as u8);
raw.extend_from_slice(b.get(p..).unwrap_or_default());
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0);
if mask != 0 {
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default();
if !mask.is_empty() {
raw.push(b'/');
raw.extend_from_slice(self.flag_chars_for(mask).as_bytes());
}
@@ -962,26 +1177,24 @@ impl AffixDict {
comments.push(c.to_string());
continue;
}
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let at = |k: usize| f.get(k).copied().unwrap_or("");
match at(0) {
"TRY" => rules.try_chars = cap_try(at(1)),
"KEY" => rules.key = at(1).to_string(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars();
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
rules.forbidden = Some(c);
} else {
rules.no_suggest = Some(c);
}
}
_ => return Err(format!("affix dictionary: bad {directive} in `{line}`")),
directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => {
let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?;
match directive {
"FORBIDDENWORD" => rules.forbidden = Some(c),
"NOSUGGEST" => rules.no_suggest = Some(c),
"SUBSTANDARD" => rules.substandard = Some(c),
_ => rules.keepcase = Some(c),
}
}
"ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),
"IGNORE" => rules.ignore = at(1).chars().collect(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
kind @ ("PFX" | "SFX") => {
let mut fc = at(1).chars();
let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) };
@@ -1223,7 +1436,7 @@ pub(crate) mod tests {
#[test]
fn more_than_64_distinct_flags_load() {
// VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin
// letter, the digits and eight accented letters: 70 flags, 64 < n <= 128.
// letter, the digits and eight accented letters: 70 flags, more than 64.
let mut flags = String::new();
let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n");
let chars: Vec<char> = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect();
@@ -1239,6 +1452,24 @@ pub(crate) mod tests {
assert!(!d.contains("verboß"), "ß is not a flag and not a stem");
}
#[test]
fn more_than_128_distinct_flags_load() {
// The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up.
let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect();
let mut aff = String::from("SET UTF-8\n");
for (i, c) in flags.chars().enumerate() {
aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n"));
}
let half: String = flags.chars().step_by(2).collect();
let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap();
assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130"));
assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("szóx198") && !back.contains("szóx199"));
let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect();
assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err());
}
#[test]
fn forbidden_words_are_rejected() {
// VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying
@@ -1364,6 +1595,167 @@ pub(crate) mod tests {
assert!(d.contains("kot") && d.contains("koty"));
}
#[test]
fn flag_aliases_stand_for_flag_sets() {
// `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags.
// Morphological aliases (`AM`) and the fields after a stem are skipped.
let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n";
let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();
assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));
assert!(!d.contains("házban"), "the stem has a and b, not c");
// An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes.
let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap();
assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined");
assert!(undefined.check("kertok"), "the other entries keep their aliases");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");
}
#[test]
fn iconv_and_ignore_normalize_input() {
// Ligatures read as their letters; `IGNORE` characters vanish from input and stems.
let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap();
assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine"));
assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)"));
assert!(!d.check("finer"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
}
#[test]
fn alias_numbers_are_read_as_hunspell_reads_them() {
// Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias
// its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags.
let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n";
let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap();
assert!(d.check("lab") && !d.check("labq") && !d.check("labs"));
assert!(d.check("falx") && !d.check("fals") && d.check("kuts"));
// A table without its count line is rejected: no alias applies.
let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap();
assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals"));
// `AF 0` is rejected the same way: `lab/B` keeps its literal flag.
let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap();
assert!(zero.check("lab") && zero.check("labs"));
// An undefined alias in a continuation class leaves the rule in force, without it.
let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap();
assert!(cont.check("haz") && cont.check("hazok"));
}
#[test]
fn a_rep_count_line_may_carry_a_comment() {
let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap();
assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]);
}
#[test]
fn count_lines_may_carry_a_comment() {
// Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`
// is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`.
let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n";
let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap();
assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`");
assert!(d.check("a1") && !d.check("a#"));
// The UTF-8 reader keeps such a line as it is, so both readers agree on it.
let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap();
assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}");
assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok"));
}
#[test]
fn dic_entries_may_hold_spaces() {
// `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.
let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";
// Checked with the Hunspell 1.7 command line: without a tab or a morphological field the
// whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it
// (` lab` is no `lab`).
let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap();
assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys"));
assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így"));
assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));
assert!(!d.contains("lab") && !d.contains("labs"));
}
#[test]
fn ignored_characters_leave_the_stripped_text_alone() {
// Hunspell removes `IGNORE` characters from the appended text, not from the stripped
// one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong,
// while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line).
let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap();
assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac"));
let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap();
assert!(add.check("ac") && add.check("acx"));
for d in [&strip, &add] {
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert_eq!(back.check("ac"), d.check("ac"));
}
}
#[test]
fn normalizing_borrows_when_nothing_changes() {
let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();
assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine")));
assert_eq!(d.normalize("(fi)ne"), "fine");
assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot")));
}
#[test]
fn keepcase_stems_match_only_as_written() {
let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap();
assert!(d.check("pH") && d.check("km") && d.check("kms"));
assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms"));
assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("km") && !back.check("Km"));
}
#[test]
fn substandard_and_lemma_present_words_are_correct() {
// `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD
// stem is never suggested; the stems and rules they mark are still correct.
let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n";
let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap();
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem"));
assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem"));
assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot"));
assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies"));
}
}
#[test]
fn unsupported_directives_are_refused_and_the_rest_load() {
// A dictionary that needs a directive outside this engine fails to load rather than
// accepting the wrong words.
for dir in UNSUPPORTED {
assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused");
}
// The directives this engine reads, with no other change to the dictionary (`AF` would
// turn the flags of `DIC` into alias numbers; it has tests of its own).
for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {
let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}"));
assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}");
}
}
#[test]
fn fields_split_on_spaces_and_tabs_only() {
// A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator.
let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n";
let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap();
assert!(d.contains("koty") && d.contains("kotem"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("koty") && back.contains("kotem"));
// The same bytes as one-flag directives, as Hunspell reads them.
let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap();
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
assert!(dict.check("kots") && !dict.check("rossz"));
assert!(dict.check("pH") && !dict.check("PH"));
}
}
#[test]
fn unsupported_and_broken_input_is_an_error() {
assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err());
+556
View File
@@ -0,0 +1,556 @@
//! Spelling suggestions from an affix dictionary, one engine for every language that reads one
//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come
//! in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`warszawa` → `Warszawa`);
//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`):
//! adding a missing accent is the commonest slip and ranks first, while taking away one the
//! writer typed ranks after the known confusions;
//! 3. the REP table of common confusions (`żeka` → `rzeka`);
//! 4. a missing space (`napewno` → `na pewno`);
//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with
//! the next or mistyped (TRY letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in
//! which a MAP slip costs half (two typos at most).
//!
//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may
//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every
//! round, so a call costs a few thousand dictionary lookups at most.
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters.
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
pub(crate) const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
pub(crate) const COST_SPLIT: u32 = 12;
/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]).
const COST_SPLIT_SHORT: u32 = 17;
/// A missing space between two longer halves with no short word beside it ([`Split::Near`]).
pub(crate) const COST_SPLIT_LONG: u32 = 18;
const COST_KEY: u32 = 13;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
pub(crate) const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
/// What a language adds to its dictionary's tables.
pub(crate) struct Tuning {
/// Where a missing space may go (round 4).
pub split: Split,
/// TRY letters for a dictionary without a TRY line.
pub alphabet: &'static str,
/// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`).
pub map: &'static [&'static str],
/// REP confusions the language adds to its dictionary's.
pub rep: &'static [(&'static str, &'static str)],
/// The cost of a REP confusion.
pub rep_cost: u32,
/// Most letters a MAP variant changes.
pub max_map_changes: usize,
/// The cost of changing letter `from` to its MAP relative `to`.
pub map_cost: fn(char, char) -> u32,
/// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead.
pub mistype_cost: fn(&[char], usize, char) -> u32,
}
/// Where a missing space may go.
pub(crate) enum Split {
/// Anywhere, dearer when a half is shorter than three letters.
Anywhere,
/// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one
/// that runs into the word before (`kocham cię`), dearer between two longer halves, and never
/// leaving a stray short half (`że ka`).
Near { leading: &'static [&'static str], trailing: &'static [&'static str] },
/// The language's own rule: the cost of the halves `a b`, or `None` to skip them.
Custom(fn(&str, &str) -> Option<u32>),
}
impl Tuning {
/// The dictionary's tables alone (Czech, Indonesian, Spanish).
pub(crate) const DEFAULT: Tuning =
Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost };
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (typing without diacritics); an accent the writer did type is likelier right,
/// so taking one away costs more than a known confusion from the REP table.
pub(crate) fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Any mistyped letter costs the same.
fn mistype_cost(_: &[char], _: usize, _: char) -> u32 {
COST_MISTYPE
}
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
/// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost).
tested: HashMap<String, u32>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input {
return;
}
match self.tested.get_mut(cand) {
Some(prev) if *prev <= cost => return,
Some(prev) => *prev = cost,
None => {
self.tested.insert(cand.to_string(), cost);
}
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
// `Mozę`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP groups: the dictionary's, then the language's.
fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec<char>]> {
if t.map.is_empty() {
return Cow::Borrowed(d.map());
}
Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect())
}
/// The other members of every MAP group of `c` (its likely confusions), each once.
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
let mut out: Vec<char> = Vec::new();
for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) {
if x != c && !out.contains(&x) {
out.push(x);
}
}
out
}
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
/// letter by letter), with its cost, each written after `buf`.
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some((&c, rest)) = chars.split_first() else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
buf.push(c);
map_variants(rest, later, t, changes, cost, buf, out);
buf.pop();
if changes < t.max_map_changes {
for &a in here {
buf.push(a);
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`.
pub(crate) fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered.
fn split_cost(split: &Split, a: &str, b: &str) -> Option<u32> {
let long = |part: &str| part.chars().count() >= 3;
match split {
Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }),
Split::Near { leading, trailing } => {
if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
Split::Custom(cost) => cost(a, b),
}
}
/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case.
pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
add_case(&mut col, word, &input);
let map = groups(d, t);
add_map_slips(&mut col, &chars, &map, t);
add_confusions(&mut col, &input, t);
add_splits(&mut col, &input, &t.split);
add_key_neighbours(&mut col, &chars);
add_typos(&mut col, &chars, t);
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars, &map);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (the first changed letter breaks ties).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let recased = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text.clone(),
};
// A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`.
let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased };
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 1: letter case.
fn add_case(col: &mut Collector, word: &str, input: &str) {
if col.d.contains(input) && word != input {
col.keep(input.to_string(), COST_CASE);
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
col.keep(capitalize(input), COST_CASE);
}
}
/// Round 2: accents and other MAP slips.
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
}
/// Round 3: common confusions.
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
for (from, to) in rep {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
}
}
}
}
/// Round 4: a missing space; both halves must be dictionary words.
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
if let Some(cost) = split_cost(split, a, b) {
col.offer(&format!("{a} {b}"), cost);
}
}
}
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
for row in col.d.key().split('|').take(32) {
let keys: Vec<char> = row.chars().take(128).collect();
for pair in keys.windows(2) {
let [a, b] = pair else { continue };
for (i, &c) in chars.iter().enumerate() {
let replacement = if c == *a {
Some(*b)
} else if c == *b {
Some(*a)
} else {
None
};
if let Some(replacement) = replacement {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = replacement;
}
col.offer(&v.iter().collect::<String>(), COST_KEY);
}
}
}
}
}
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
/// equal-cost candidates come out in that order.
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
let d = col.d;
let n = chars.len();
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
let mut letters: Vec<char> = Vec::new();
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
if !letters.contains(&c) {
letters.push(c);
}
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.to_vec();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.to_vec();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.to_vec();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
}
}
}
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
let d = col.d;
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT);
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
}
/// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop)
/// ranks at the lower one.
#[test]
fn a_cheaper_later_offer_wins() {
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
col.offer("cafe", COST_DROP_DIACRITIC);
col.offer("cafe", COST_REP);
col.offer("cafe", COST_SCAN);
assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP));
}
#[test]
fn distance_weighs_map_slips_half() {
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
/// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged
/// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form.
#[test]
fn keepcase_suggestions_are_not_recased_into_errors() {
let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap();
assert!(!d.check("Km") && !d.check("KM"));
for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] {
let got = suggest_in(&d, word, 4, &Tuning::DEFAULT);
assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}");
assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}");
}
assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::<String>::new());
assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies"));
}
/// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY
/// line, and a split rule decides which missing spaces are offered.
#[test]
fn tuning_adds_to_the_dictionary_tables() {
let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap();
let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT };
assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into()));
assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group");
assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair");
assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters");
let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT };
assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into()));
assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half");
assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into()));
}
}
+10 -75
View File
@@ -7,88 +7,35 @@ pub(crate) mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Czech Hunspell dictionary.
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
/// The Czech dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Czech");
/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.dict.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Czech spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Czech dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals
@@ -112,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Czech dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+5 -389
View File
@@ -1,168 +1,8 @@
//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits
//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian
//! ([`crate::id`]) uses the same ranking with its own dictionary.
//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed
//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters (Spanish shares the bound).
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 17;
const COST_KEY: u32 = 13;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (a capitalized name).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (typing without diacritics); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
use crate::affix::AffixDict;
use crate::affix_suggest::Tuning;
/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -171,237 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
let map = d.map();
add_case_suggestions(&mut col, &input, word);
add_map_suggestions(&mut col, &chars, map);
add_replacement_suggestions(&mut col, &input, d.rep());
add_split_suggestions(&mut col, &input);
add_keyboard_suggestions(&mut col, &chars, d.key());
add_typing_suggestions(&mut col, &chars, d.try_chars());
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (the first changed letter breaks ties).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
fn add_case_suggestions(col: &mut Collector, input: &str, word: &str) {
if col.d.contains(input) && word != input {
col.keep(input.to_string(), COST_CASE);
} else {
let capitalized = capitalize(input);
if col.d.contains(&capitalized) && word != capitalized {
col.keep(capitalized, COST_CASE);
}
}
}
fn add_map_suggestions(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
let alternatives: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(chars, &alternatives, 0, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
variants.sort_by_key(|variant| variant.1);
for (variant, cost) in variants {
col.offer(&variant, cost);
}
}
fn add_replacement_suggestions(col: &mut Collector, input: &str, replacements: &[(String, String)]) {
for (from, to) in replacements {
if from.is_empty() {
continue;
}
for (index, _) in input.match_indices(from) {
if let (Some(prefix), Some(suffix)) = (input.get(..index), input.get(index + from.len()..)) {
col.offer(&format!("{prefix}{to}{suffix}"), COST_REP);
}
}
}
}
fn add_split_suggestions(col: &mut Collector, input: &str) {
for (index, _) in input.char_indices().skip(1) {
let (Some(first), Some(second)) = (input.get(..index), input.get(index..)) else { continue };
let cost = if first.chars().count() >= 3 && second.chars().count() >= 3 { COST_SPLIT } else { COST_SPLIT_RARE };
col.offer(&format!("{first} {second}"), cost);
}
}
fn add_keyboard_suggestions(col: &mut Collector, chars: &[char], key_rows: &str) {
for row in key_rows.split('|').take(32) {
let keys: Vec<char> = row.chars().take(128).collect();
for pair in keys.windows(2) {
let [first, second] = pair else { continue };
for (index, &character) in chars.iter().enumerate() {
let replacement = if character == *first {
Some(*second)
} else if character == *second {
Some(*first)
} else {
None
};
if let Some(replacement) = replacement {
let mut candidate = chars.to_vec();
if let Some(character) = candidate.get_mut(index) {
*character = replacement;
}
col.offer(&candidate.iter().collect::<String>(), COST_KEY);
}
}
}
}
}
fn add_typing_suggestions(col: &mut Collector, chars: &[char], try_chars: &str) {
let mut letters: Vec<char> = try_chars.chars().filter(|character| character.is_lowercase()).take(128).collect();
letters.sort_unstable();
letters.dedup();
let word = |candidate: &[char]| candidate.iter().collect::<String>();
add_swap_suggestions(col, chars, &word);
add_drop_suggestions(col, chars, &word);
add_insert_suggestions(col, chars, &letters, &word);
add_mistype_suggestions(col, chars, &letters, &word);
}
fn add_swap_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len().saturating_sub(1) {
if chars.get(index) != chars.get(index + 1) {
let mut candidate = chars.to_vec();
candidate.swap(index, index + 1);
col.offer(&word(&candidate), COST_SWAP);
}
}
}
fn add_drop_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len() {
let mut candidate = chars.to_vec();
candidate.remove(index);
col.offer(&word(&candidate), COST_DROP_OR_ADD);
}
}
fn add_insert_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..=chars.len() {
for &letter in letters {
let mut candidate = chars.to_vec();
candidate.insert(index, letter);
col.offer(&word(&candidate), COST_DROP_OR_ADD);
}
}
}
fn add_mistype_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len() {
for &letter in letters {
if chars.get(index) != Some(&letter) {
let mut candidate = chars.to_vec();
if let Some(character) = candidate.get_mut(index) {
*character = letter;
}
col.offer(&word(&candidate), COST_MISTYPE);
}
}
}
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
let suggestions = suggest_in(&d, "počítačz", 8);
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
}
#[test]
fn suggestions_are_fast_enough() {
crate::cs::test_dict::install();
+21 -76
View File
@@ -16,16 +16,19 @@ mod suggest;
pub(crate) mod test_dict;
pub mod voseo;
use std::sync::{Arc, PoisonError, RwLock};
use std::sync::Arc;
use crate::ProofLang;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first
/// use, then the installed one (or an empty one).
static DICTS: [RwLock<Option<Arc<AffixDict>>>; 4] = [const { RwLock::new(None) }; 4];
/// The dictionary of each variety, found on first use (see [`DictSlot`]).
pub(crate) static SPAIN_DICT: DictSlot<AffixDict> = DictSlot::new(SPAIN, "Spanish (Spain)");
pub(crate) static MEXICO_DICT: DictSlot<AffixDict> = DictSlot::new(MEXICO, "Spanish (Mexico)");
pub(crate) static LATIN_AMERICA_DICT: DictSlot<AffixDict> = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)");
pub(crate) static URUGUAY_DICT: DictSlot<AffixDict> = DictSlot::new(URUGUAY, "Spanish (Uruguay)");
/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the
/// generic one RLA-ES publishes).
@@ -40,84 +43,38 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e
/// or generic one.
const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"];
/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's
/// file names; Spain for any other language.
fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) {
/// `lang`'s variety's dictionary; Spain's for any other language.
fn slot(lang: ProofLang) -> &'static DictSlot<AffixDict> {
match lang {
ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO),
ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA),
ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY),
_ => (0, "Spanish (Spain)", SPAIN),
ProofLang::EsMx => &MEXICO_DICT,
ProofLang::Es419 => &LATIN_AMERICA_DICT,
ProofLang::EsUy => &URUGUAY_DICT,
_ => &SPAIN_DICT,
}
}
/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`,
/// `es_UY`): the first of its file names, used for the status-bar hint and load errors.
pub fn dictionary_name(lang: ProofLang) -> &'static str {
variety(lang).2.first().copied().unwrap_or("es_ES")
slot(lang).name()
}
/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given
/// with [`install_dictionary`], or an empty one when none is available (spelling is off then;
/// see [`has_dictionary`]).
pub fn dictionary(lang: ProofLang) -> Arc<AffixDict> {
let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) };
if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed(lang));
let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed(lang: ProofLang) -> AffixDict {
let (_, name, files) = variety(lang);
let key = dictionary_name(lang);
let Some((aff, dic)) = crate::hunspell::find(files) else {
crate::hunspell::clear_load_error(key);
log::info!("{name} spelling: no Hunspell {key} dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("{name} spelling: {e}");
AffixDict::default()
})
slot(lang).get()
}
/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text,
/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
if let Some(slot) = DICTS.get(variety(lang).0) {
*slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d));
}
Ok(())
}
/// Forget every variety's loaded dictionary, so the next check looks for an installed one again
/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
for slot in &DICTS {
*slot.write().unwrap_or_else(PoisonError::into_inner) = None;
}
}
/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder
/// or the system locations now (File › Options › Proofing shows it); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path(lang: ProofLang) -> Option<std::path::PathBuf> {
crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic)
slot(lang).install(aff, dic)
}
/// Is a dictionary available for `lang`'s variety (spelling is checked)?
pub fn has_dictionary(lang: ProofLang) -> bool {
!dictionary(lang).is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
slot(lang).has_dictionary()
}
/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in
@@ -143,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Spanish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang));
}
@@ -200,7 +145,7 @@ mod tests {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
for lang in VARIETIES {
let found = crate::hunspell::find_in(&dir, variety(lang).2);
let found = crate::hunspell::find_in(&dir, slot(lang).names());
assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}");
if let Some((aff, dic)) = found {
assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang));
@@ -215,7 +160,7 @@ mod tests {
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
}
for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] {
assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
assert_eq!(dictionary_name(lang), name);
}
std::fs::remove_dir_all(dir).unwrap();
+4 -4
View File
@@ -1,5 +1,5 @@
//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses
//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to
//! Spanish suggestions. The ranking is the shared dictionary-driven one
//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to
//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces,
//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary.
//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first
@@ -7,7 +7,7 @@
use crate::ProofLang;
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::cs::suggest::MAX_LETTERS;
use crate::affix_suggest::{MAX_LETTERS, Tuning};
/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's
/// case.
@@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec
_ => v,
});
}
for s in crate::cs::suggest::suggest_in(d, word, max) {
for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) {
if !out.contains(&s) {
out.push(s);
}
+10 -60
View File
@@ -9,73 +9,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Croatian Hunspell dictionary.
const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"];
/// The Croatian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Croatian");
/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Croatian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Croatian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals
@@ -99,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Croatian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+25 -327
View File
@@ -1,12 +1,11 @@
//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance,
//! missing spaces and a bounded search. Proper Croatian corrections depend on
//! the actual installed dictionary entries.
//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of
//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines.
use std::collections::{HashMap, HashSet};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`).
const LEADING_WORDS: &[&str] = &[
"a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će",
"bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad",
@@ -14,159 +13,13 @@ const LEADING_WORDS: &[&str] = &[
const TRAILING_WORDS: &[&str] =
&["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table
/// (`seje` → `seja` before `sejé`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// What Croatian adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiourntsjkvlmdpbgczčćšžđfh",
map: &["cčć", "dđ", "sš", "zž"],
..Tuning::DEFAULT
};
/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -175,179 +28,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']];
let map = if d.map().is_empty() { &fallback_map } else { d.map() };
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (`saúde` before `saudê`: both add an accent, but at the first difference).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
/// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća`
/// first, before a typo that costs more (`kuha`).
#[test]
fn diacritics_without_map_lines() {
let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap();
assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]);
}
#[test]
fn croatian_suggestions() {
crate::hr::test_dict::install();
for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] {
+340 -4
View File
@@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
///
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
/// its number (the Turkish dictionary's `FLAG num`).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
let (set, flag) = directives(aff);
let set = set.unwrap_or_else(|| "UTF-8".into());
match set.as_str() {
"UTF-8" | "UTF8" => {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
}
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
@@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
}
}
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
let (mut set, mut flag) = (None, None);
for l in aff.split(|&b| b == b'\n') {
if set.is_some() && flag.is_some() {
break;
}
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
let slot = match l.first() {
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
_ => continue,
};
let arg = l
.strip_prefix(slot.1)
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
if let Some(a) = arg {
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
}
}
(set, flag)
}
/// Directives whose one argument is a flag.
const FLAG_DIRECTIVES: &[&str] = &[
"CIRCUMFIX",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFLAG",
"COMPOUNDFORBIDFLAG",
"COMPOUNDLAST",
"COMPOUNDFIRST",
"COMPOUNDMIDDLE",
"COMPOUNDPERMITFLAG",
"COMPOUNDROOT",
"FORBIDDENWORD",
"FORCEUCASE",
"GENERATE",
"HU_KOTOHANGZO",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"NOSUGGEST",
"ONLYINCOMPOUND",
"ONLYROOT",
"PSEUDOROOT",
"SUBSTANDARD",
"WARN",
];
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
/// names the line in an error.
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
i += 2;
continue;
}
if b == b'/' {
slash = Some(i);
break;
}
i += 1;
}
let (text, flags) = match slash {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
out.push_str(&decode_latin1(flags));
Ok(out)
}
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
/// its parts is the original).
fn fields(line: &[u8]) -> Vec<&[u8]> {
let mut out = Vec::new();
let mut start = 0;
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
for (i, b) in line.iter().enumerate() {
let sep = matches!(b, b' ' | b'\t');
if sep != in_sep {
out.push(line.get(start..i).unwrap_or_default());
start = i;
in_sep = sep;
}
}
out.push(line.get(start..).unwrap_or_default());
out
}
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(aff.len());
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.is_ascii()
{
out.push_str(s);
continue;
}
let parts = fields(line);
// Field index among the non-separator parts.
let words: Vec<usize> =
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
let dir = words
.first()
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = || format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
}
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
// a flag; from there on the line is kept as text. It can only start after the directive's
// own fields, since a field such as an affix condition may itself begin with `#`.
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
let first_free = comment_field(&dir, field_text);
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
// The field index of the next non-separator part.
let mut next_field = 0;
for (i, p) in parts.iter().enumerate() {
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
next_field += 1;
next_field - 1
});
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
};
out.push_str(&decoded);
}
}
Ok(out)
}
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
match dir {
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
"PFX" | "SFX" => 5,
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
{
2
}
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
"CHECKCOMPOUNDPATTERN" => 4,
_ => 2,
}
}
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
let blank = |b: &u8| matches!(b, b' ' | b'\t');
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
let tab = line.iter().position(|&b| b == b'\t');
let end = match (morph, tab) {
(Some(m), Some(t)) => m.min(t),
(m, t) => match m.or(t) {
Some(e) => e,
None => line.len(),
},
};
// Blanks before the morphological field or the line end aren't part of the entry.
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
}
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(dic.len());
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
{
out.push_str(s);
continue;
}
let what = || format!(".dic line {}", n + 1);
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
// morphological fields after it are UTF-8 text.
let end = dic_entry_end(line);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -280,6 +509,78 @@ mod tests {
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn utf8_dictionaries_keep_single_byte_flags() {
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
let (a, d) = decode(aff, dic).unwrap();
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
assert!(!dict.check("házban"));
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
}
#[test]
fn single_flag_fields_keep_their_first_byte() {
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
assert!(d.contains("kert/\u{c3}\u{a9}"));
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("kert") && dict.check("kertok"));
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
}
#[test]
fn numeric_and_long_flags_stay_text() {
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
// number with a non-ASCII rule or comment keeps every digit.
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
assert_eq!(a, aff);
assert_eq!(d, "1\nkitap/1234\n");
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
assert!(tr.contains("kitapları") && tr.contains("kitap"));
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
// `FLAG DEFAULT` is the single-byte default.
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
assert_eq!(d, "1\nkert/\u{e9}\n");
}
#[test]
fn flag_utf8_counts_anywhere_in_the_aff() {
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
for _ in 0..400 {
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
}
aff.extend_from_slice(b"FLAG UTF-8\r\n");
assert!(aff.len() > 8192);
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
}
#[test]
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
@@ -291,6 +592,41 @@ mod tests {
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
}
#[test]
fn comments_start_after_the_directive_fields() {
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
assert_eq!(comment_field("REP", f("REP 12")), 2);
assert_eq!(comment_field("REP", f("REP # x")), 3);
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
}
#[test]
fn dic_entries_end_where_hunspell_ends_them() {
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
assert_eq!(end("ad hoc/B "), "ad hoc/B");
assert_eq!(end("word"), "word");
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
// Without a tab or a morphological field the whole line is the entry: its byte flags
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\n");
}
#[test]
fn libreoffice_extension_dirs_follow_the_language() {
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
+12 -61
View File
@@ -9,78 +9,41 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
/// Java locales).
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
/// The Indonesian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Indonesian");
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Indonesian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it); `None` when there is none. A
/// dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is an Indonesian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Indonesian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return known(d, w)
|| w.split('-').all(|p| {
+4 -5
View File
@@ -1,7 +1,6 @@
//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses
//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem
//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is
//! hard-coded here.
//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing
//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID`
//! dictionary, so nothing Indonesian is hard-coded here.
use crate::affix::AffixDict;
@@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::cs::suggest::suggest_in(d, word, max)
crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT)
}
#[cfg(test)]
+10 -59
View File
@@ -18,72 +18,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Italian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
/// The Italian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Italian");
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Italian spelling: no Hunspell it_IT dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Italian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is an Italian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
@@ -125,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Italian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
let lower = w.to_lowercase();
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
return true;
+16 -336
View File
@@ -1,25 +1,9 @@
//! Italian spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`roma` → `Roma`);
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`,
//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`);
//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which an accent slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
//! dictionary lookups at most.
//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and
//! letters to try for a dictionary without a TRY line.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
const LEADING_WORDS: &[&str] = &[
@@ -35,158 +19,12 @@ const TRAILING_WORDS: &[&str] = &[
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
"è",
];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance.
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
/// What Italian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `à` → `a`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -195,182 +33,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent.
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn italian_misspellings_get_the_right_suggestion() {
crate::it::test_dict::install();
+197 -135
View File
@@ -50,6 +50,7 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
pub mod affix;
mod affix_suggest;
pub mod bg;
pub mod cs;
pub mod dict;
@@ -65,6 +66,8 @@ pub mod patterns;
pub mod pl;
pub mod pt;
pub mod ru;
mod slot;
mod spelling;
pub mod tr;
pub mod vi;
@@ -157,57 +160,150 @@ pub fn ignore_word(w: &str) {
}
}
/// A language proofed against an installed Hunspell dictionary: all the functions below need to
/// know about it, so a language joins with one row in [`LANGUAGES`].
struct Language {
lang: ProofLang,
slot: &'static dyn slot::Slot,
/// The spelling mark's message, in the language.
message: &'static str,
is_correct: fn(&str, ProofLang, &Options) -> bool,
suggest: fn(&str, usize, ProofLang) -> Vec<String>,
/// Grammar issues; the flag says whether the first word must have a capital (not in a list
/// item).
grammar: fn(&str, ProofLang, bool) -> Vec<Issue>,
}
/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing
/// lists them. English has a built-in word list and is handled apart.
static LANGUAGES: [Language; 12] = [
Language {
lang: ProofLang::Pl,
slot: &pl::DICT,
message: "Możliwy błąd pisowni",
is_correct: |w, _, o| pl::is_correct_with_options(w, o),
suggest: |w, n, _| pl::suggest(w, n),
grammar: |t, _, c| pl::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Pt,
slot: &pt::DICT,
message: "Possível erro de ortografia",
is_correct: |w, _, o| pt::is_correct_with_options(w, o),
suggest: |w, n, _| pt::suggest(w, n),
grammar: |t, _, c| pt::grammar::check_with(t, c),
},
Language {
lang: ProofLang::It,
slot: &it::DICT,
message: "Possibile errore di ortografia",
is_correct: |w, _, o| it::is_correct_with_options(w, o),
suggest: |w, n, _| it::suggest(w, n),
grammar: |t, _, c| it::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Hr,
slot: &hr::DICT,
message: "Moguća pravopisna pogreška",
is_correct: |w, _, o| hr::is_correct_with_options(w, o),
suggest: |w, n, _| hr::suggest(w, n),
grammar: |t, _, c| hr::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Tr,
slot: &tr::DICT,
message: "Olası yazım hatası",
is_correct: |w, _, o| tr::is_correct_with_options(w, o),
suggest: |w, n, _| tr::suggest(w, n),
grammar: |t, _, c| tr::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Ru,
slot: &ru::DICT,
message: "Возможна орфографическая ошибка",
is_correct: |w, _, o| ru::is_correct_with_options(w, o),
suggest: |w, n, _| ru::suggest(w, n),
grammar: |t, _, c| ru::grammar::check_with(t, c),
},
// Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no
// exception.
Language {
lang: ProofLang::Cs,
slot: &cs::DICT,
message: "Možná pravopisná chyba",
is_correct: |w, _, o| cs::is_correct_with_options(w, o),
suggest: |w, n, _| cs::suggest(w, n),
grammar: |t, _, _| cs::grammar::check(t),
},
Language {
lang: ProofLang::Id,
slot: &id::DICT,
message: "Kemungkinan salah eja",
is_correct: |w, _, o| id::is_correct_with_options(w, o),
suggest: |w, n, _| id::suggest(w, n),
grammar: |t, _, _| id::grammar::check(t),
},
Language {
lang: ProofLang::Es,
slot: &es::SPAIN_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::EsMx,
slot: &es::MEXICO_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::Es419,
slot: &es::LATIN_AMERICA_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::EsUy,
slot: &es::URUGUAY_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
];
/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof.
fn language(lang: ProofLang) -> Option<&'static Language> {
LANGUAGES.iter().find(|l| l.lang == lang)
}
/// Search `dir` for Hunspell dictionaries before the system locations (File › Options ›
/// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from
/// the new search path, and text is checked again.
pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
hunspell::set_dictionary_dir(dir);
pl::reload_dictionary();
pt::reload_dictionary();
it::reload_dictionary();
hr::reload_dictionary();
tr::reload_dictionary();
ru::reload_dictionary();
cs::reload_dictionary();
id::reload_dictionary();
es::reload_dictionary();
for l in &LANGUAGES {
l.slot.reload();
}
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
}
/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it
/// now (`None`: not installed, so the language has no spelling marks).
pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)> {
vec![
(ProofLang::Pl, pl::installed_path()),
(ProofLang::Pt, pt::installed_path()),
(ProofLang::It, it::installed_path()),
(ProofLang::Hr, hr::installed_path()),
(ProofLang::Tr, tr::installed_path()),
(ProofLang::Ru, ru::installed_path()),
(ProofLang::Cs, cs::installed_path()),
(ProofLang::Id, id::installed_path()),
(ProofLang::Es, es::installed_path(ProofLang::Es)),
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
(ProofLang::Es419, es::installed_path(ProofLang::Es419)),
(ProofLang::EsUy, es::installed_path(ProofLang::EsUy)),
]
LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect()
}
/// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse
/// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has
/// checked yet reports `None` (#2830).
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
let key = match lang {
ProofLang::Pl => "pl_PL",
ProofLang::Pt => "pt_BR",
ProofLang::It => "it_IT",
ProofLang::Hr => "hr_HR",
ProofLang::Tr => "tr_TR",
ProofLang::Id => "id_ID",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang),
_ => return None,
};
hunspell::load_error(key).map(|(_, reason)| reason)
hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason)
}
/// A number that changes whenever a word is added to or removed from the user dictionary, or the
@@ -438,36 +534,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p
/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English).
fn dictionary_loaded(lang: ProofLang) -> Option<bool> {
Some(match lang {
ProofLang::Pl => pl::has_dictionary(),
ProofLang::Pt => pt::has_dictionary(),
ProofLang::It => it::has_dictionary(),
ProofLang::Hr => hr::has_dictionary(),
ProofLang::Tr => tr::has_dictionary(),
ProofLang::Ru => ru::has_dictionary(),
ProofLang::Cs => cs::has_dictionary(),
ProofLang::Id => id::has_dictionary(),
l if l.is_spanish() => es::has_dictionary(l),
_ => return None,
})
language(lang).map(|l| l.slot.has_dictionary())
}
/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for;
/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`).
fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> {
match lang {
_ if installed => None,
ProofLang::Pl => Some("pl_PL"),
ProofLang::Pt => Some("pt_BR"),
ProofLang::It => Some("it_IT"),
ProofLang::Hr => Some("hr_HR"),
ProofLang::Tr => Some("tr_TR"),
ProofLang::Ru => Some("ru_RU"),
ProofLang::Cs => Some("cs_CZ"),
ProofLang::Id => Some("id_ID"),
l if l.is_spanish() => Some(es::dictionary_name(l)),
_ => None,
}
language(lang).filter(|_| !installed).map(|l| l.slot.name())
}
/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary
@@ -739,18 +812,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) {
return None;
}
let message = match lang {
ProofLang::Pl => "Możliwy błąd pisowni",
ProofLang::Hr => "Moguća pravopisna pogreška",
ProofLang::Pt => "Possível erro de ortografia",
ProofLang::It => "Possibile errore di ortografia",
ProofLang::Tr => "Olası yazım hatası",
ProofLang::Ru => "Возможна орфографическая ошибка",
ProofLang::Cs => "Možná pravopisná chyba",
ProofLang::Id => "Kemungkinan salah eja",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico",
_ => "Possible spelling mistake",
};
let message = language(lang).map_or("Possible spelling mistake", |l| l.message);
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
}
@@ -774,40 +836,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> {
/// checked as its composed form ([`composed`]).
pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
let word = &*composed(word);
match lang {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts),
ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts),
ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts),
ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts),
ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts),
ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word),
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => {
accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts)
}
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts),
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts),
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
if lang.is_english() {
return is_correct_english(word, lang, opts);
}
// A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks.
language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts))
}
/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed
/// ([`composed`]) like the dictionaries' words.
pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
let word = &*composed(word);
match lang {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang),
ProofLang::Pl => pl::suggest(word, max),
ProofLang::Hr => hr::suggest(word, max),
ProofLang::Pt => pt::suggest(word, max),
ProofLang::It => it::suggest(word, max),
ProofLang::Tr => tr::suggest(word, max),
ProofLang::Ru => ru::suggest(word, max),
ProofLang::Cs => cs::suggest(word, max),
ProofLang::Id => id::suggest(word, max),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
if lang.is_english() {
return suggest_english(word, max, lang);
}
language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang))
}
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
@@ -824,18 +867,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
match lang {
ProofLang::En => grammar_en(text, capital_first),
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
ProofLang::Pl => pl::grammar::check_with(text, capital_first),
ProofLang::Hr => hr::grammar::check_with(text, capital_first),
ProofLang::Pt => pt::grammar::check_with(text, capital_first),
ProofLang::It => it::grammar::check_with(text, capital_first),
ProofLang::Tr => tr::grammar::check_with(text, capital_first),
ProofLang::Ru => ru::grammar::check_with(text, capital_first),
// Czech has no sentence-capital rule, so a list item needs no exception.
ProofLang::Cs => cs::grammar::check(text),
ProofLang::Id => id::grammar::check(text),
// Spanish has no sentence-capital rule either.
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
_ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)),
}
}
@@ -1060,7 +1092,9 @@ mod tests {
use super::*;
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test.
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land
/// mid-test, and by those that save the whole user dictionary and load it back, which would
/// bring back a word another test just removed.
static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// #1063: the dictionaries folder is searched before the system locations; the dictionaries
@@ -1077,6 +1111,7 @@ mod tests {
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC),
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
@@ -1096,41 +1131,67 @@ mod tests {
let before = dictionary_generation();
set_dictionaries_folder(Some(dir.clone()));
assert!(dictionary_generation() > before, "cached checks are dropped");
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
// Every language is listed, Russian and Czech too (#3226: they were once missing, so
// neither showed up in File › Options › Proofing). A language reports the folder's file,
// or none while a dictionary given with install_dictionary() is in use (#3252 review);
// the languages' own tests install theirs concurrently, so an exact path would race them.
// The folder is searched first whatever is in use.
let found = installed_dictionaries();
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"];
assert_eq!(
found[7..],
found.iter().map(|(lang, _)| *lang).collect::<Vec<_>>(),
[
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
(ProofLang::EsMx, Some(dir.join("es_MX.dic"))),
(ProofLang::Es419, Some(dir.join("es_419.dic"))),
(ProofLang::EsUy, Some(dir.join("es_UY.dic"))),
ProofLang::Pl,
ProofLang::Pt,
ProofLang::It,
ProofLang::Hr,
ProofLang::Tr,
ProofLang::Ru,
ProofLang::Cs,
ProofLang::Id,
ProofLang::Es,
ProofLang::EsMx,
ProofLang::Es419,
ProofLang::EsUy
]
);
assert_eq!(
found[..4],
[
(ProofLang::Pl, Some(dir.join("pl_PL.dic"))),
(ProofLang::Pt, Some(dir.join("pt_BR.dic"))),
(ProofLang::It, Some(dir.join("it_IT.dic"))),
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
]
);
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
// installed_path() in either module), so neither ever showed up in File › Options ›
// Proofing, installed or not. Their languages, not their exact path, is what belongs
// here: unlike the four above, installed_path() for Russian and Czech also reflects
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
// test modules do, concurrently, in tests of their own — asserting an exact path here
// would race them for no reason this test needs to care about.
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
for ((lang, path), name) in found.iter().zip(names) {
let file = dir.join(format!("{name}.dic"));
assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}");
assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}");
}
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
}
/// A dictionary that fails to load says why in File › Options › Proofing in every language
/// that reads one; Russian and Czech were missing from the list that named their dictionaries
/// there.
#[test]
fn every_dictionary_language_reports_its_load_error() {
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
for (lang, _) in installed_dictionaries() {
let name = dictionary_needed(lang, false).unwrap();
let reason = format!("zq unreadable {name}");
let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone()));
assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}");
hunspell::clear_load_error(name);
}
}
/// Every proofed language other than English has a row in the language table, and only those.
#[test]
fn the_language_table_covers_the_proofed_languages() {
for l in ProofLang::PROOFED {
assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}");
}
for (i, row) in LANGUAGES.iter().enumerate() {
assert!(row.lang.is_proofed(), "{:?}", row.lang);
assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang);
}
}
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
/// with the rule; a plain paragraph's does, and later sentences in the item are still checked.
#[test]
@@ -1593,6 +1654,7 @@ mod tests {
/// #40: the user dictionary survives a save and a fresh load.
#[test]
fn user_dictionary_persists_through_a_file() {
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist"));
let _ = std::fs::remove_dir_all(&dir);
// The folder is created on the first save.
+14 -63
View File
@@ -17,78 +17,41 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Polish dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Polish Hunspell dictionary.
const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"];
/// The Polish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Polish");
/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then;
/// see [`has_dictionary`]).
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
DICT.get()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Polish spelling: no Hunspell pl_PL dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Polish spelling: {e}");
AffixDict::default()
})
}
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
/// for example one the user picked. On error the current dictionary stays.
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Polish dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced
/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`).
const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"];
/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals
/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so
/// is every word when no Polish dictionary is installed.
@@ -110,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Polish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
let w = w.replace('’', "'");
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
+19 -342
View File
@@ -1,26 +1,9 @@
//! Polish spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`warszawa` → `Warszawa`);
//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy`
//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most
//! common slip, so adding them ranks first, while taking away a diacritic the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` →
//! `huśtawka`, `pszyjaciel` → `przyjaciel`);
//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham
//! cię`), cheaper when one half is a short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which a diacritic slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand
//! dictionary lookups at most.
//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`),
//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`).
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning};
/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede
/// wszystkim`).
@@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[
/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`).
const TRAILING_WORDS: &[&str] =
&["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
/// What Polish adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx",
mistype_cost,
..Tuning::DEFAULT
};
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
// `Mozę`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the
/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the
/// writer did type is likelier right, so taking one away costs more than a known confusion from
/// the REP table (`żeka` → `rzeka` before `zeka`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it
/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo.
fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 {
let last = i + 1 == chars.len();
if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }
}
/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case.
@@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Diacritics and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half (`że ka`).
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
// Final devoicing can make a voiced consonant sound voiceless.
let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g'));
let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE };
col.offer(&s(&v), cost);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
found.sort_by_key(|(_, k)| *k);
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
+10 -59
View File
@@ -16,72 +16,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Portuguese Hunspell dictionary.
const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"];
/// The Portuguese dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Portuguese");
/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Portuguese spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Portuguese dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Portuguese dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+13 -343
View File
@@ -1,26 +1,9 @@
//! Portuguese spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`brasília` → `Brasília`);
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`,
//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` →
//! `exceção`, `previlégio` → `privilégio`);
//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which an accent slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
//! dictionary lookups at most.
//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and
//! letters to try for a dictionary without a TRY line.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
const LEADING_WORDS: &[&str] = &[
@@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
"eu", "tu", "que", "de", "em", "é",
];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table
/// (`seje` → `seja` before `sejé`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// What Portuguese adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw",
..Tuning::DEFAULT
};
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (`saúde` before `saudê`: both add an accent, but at the first difference).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
+51 -66
View File
@@ -28,83 +28,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Russian Hunspell dictionary.
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
/// The Russian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Russian");
/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
/// see [`has_dictionary`]).
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.dict.get_or_insert(loaded).clone()
DICT.get()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Russian spelling: no Hunspell ru_RU dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Russian spelling: {e}");
AffixDict::default()
})
}
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
/// for example one the user picked. On error the current dictionary stays.
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Russian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)?
@@ -130,18 +82,30 @@ pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits or superscripts.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Russian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
let w = w.trim_matches('-');
let n = w.chars().count();
if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) {
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
return true;
}
if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
return true;
}
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
@@ -161,7 +125,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
return true;
}
if w.contains('-') {
return hyphenated(d, w);
return hyphenated(d, w, opts);
}
solid_compound(d, w)
}
@@ -340,7 +304,7 @@ fn known(d: &AffixDict, w: &str) -> bool {
/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles,
/// prefixes and linking vowels Russian joins with a hyphen.
fn hyphenated(d: &AffixDict, w: &str) -> bool {
fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
let parts: Vec<&str> = w.split('-').collect();
if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 {
return false;
@@ -353,7 +317,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
&& lp.iter().skip(k).all(|p| PARTICLES.contains(p))
{
let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default();
return known(d, &base) || (base.contains('-') && hyphenated(d, &base));
return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts));
}
match lp.as_slice() {
// `кое-кто`, `кой-какой`.
@@ -369,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
let last = parts.len() - 1;
parts.iter().enumerate().all(|(i, p)| {
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
return true;
}
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
@@ -497,4 +461,25 @@ mod tests {
crate::add_word("Зкщфрусскоеслово");
assert!(is_correct("зкщфрусскоеслово"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_russian_check() {
test_dict::install();
let lang = crate::ProofLang::Ru;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default");
assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults));
assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too");
assert!(crate::is_correct_in_with("превет2", lang, &defaults));
assert!(!crate::is_correct_in_with("превет2", lang, &check_digits));
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
}
}
+70 -334
View File
@@ -1,30 +1,14 @@
//! Russian spelling suggestions, best first.
//!
//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the
//! common confusions are defined here. Candidates come in rounds of rising cost, each checked
//! against the dictionary:
//! 1. letter case (`москва` → `Москва`);
//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`,
//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to
//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`);
//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or
//! doubled consonant, `здраствуй` → `здравствуй`);
//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped;
//! 6. when that finds fewer than asked for: the stems that start like the word and share the
//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within
//! a letter group costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round.
//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP`
//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` →
//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule
//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`).
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning};
/// Letters confused with each other: a slip inside a group is cheaper than another typo.
const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"];
/// Common confusions: a written sequence and what was meant.
const REP: &[(&str, &str)] = &[
("жы", "жи"),
@@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[
("ц", "тс"),
("тс", "ц"),
];
/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`).
const LEADING_WORDS: &[&str] = &[
"не",
@@ -123,158 +106,42 @@ const PREPOSITIONS: &[&str] = &[
const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"];
/// The Russian alphabet, when the dictionary has no `TRY` line.
const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё";
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 3;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
/// `е` written for `ё`, or the other way round, is barely a slip.
const COST_YO: u32 = 3;
const COST_MAP: u32 = 7;
const COST_REP: u32 = 9;
const COST_SPLIT: u32 = 12;
/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`).
const COST_SPLIT_PREPOSITION: u32 = 17;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
/// What Russian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT };
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance.
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The other members of each [`MAP`] group holding `c`.
fn alternatives(c: char) -> Vec<char> {
let mut out: Vec<char> = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect();
out.sort_unstable();
out.dedup();
out
}
/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip
/// (`ё` is optional in ordinary writing).
/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest.
fn map_cost(from: char, to: char) -> u32 {
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP }
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP }
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
/// A missing space: cheap after a short leading word or before a short trailing one, dearer after
/// a short preposition or between two longer words, never leaving a stray short half.
fn split(a: &str, b: &str) -> Option<u32> {
let long = |p: &str| p.chars().count() >= 3;
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
let preposition = PREPOSITIONS.contains(&a);
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
return None;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
Some(COST_SPLIT_PREPOSITION)
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
/// A letter's group representative (the first of its first group), lower case: `ё` → `е`,
/// `з` → `з`, `с` → `з`.
fn fold(c: char) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l)
}
/// Optimal string alignment distance where a slip within a letter group costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if fold(ai) == fold(bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
@@ -282,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Slips within a letter group.
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in REP {
for (i, _) in input.match_indices(from) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
let preposition = PREPOSITIONS.contains(&a);
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
continue;
}
let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
COST_SPLIT_PREPOSITION
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = ALPHABET.chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
found.sort_by_key(|(_, k)| *k);
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a group relative of it,
/// either case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(fold).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
@@ -458,6 +161,36 @@ mod tests {
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
/// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip
/// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip;
/// a preposition is never split off a verb, and after a one-letter one a typo is likelier.
#[test]
fn russian_costs_rank_its_slips() {
let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap();
assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]);
assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]);
assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into()));
let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4);
assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}");
assert!(s.contains(&"в доме".into()), "{s:?}");
}
/// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never
/// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the
/// shared Latin-script check doesn't reach it.
#[test]
fn nosuggest_words_are_never_offered() {
let offered = |aff: &str, input: &str| {
let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap();
suggest_in(&d, input, 16)
};
assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared");
for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] {
let s = offered("SET UTF-8\nNOSUGGEST N\n", input);
assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}");
}
}
#[test]
fn russian_misspellings_get_the_right_suggestion() {
crate::ru::test_dict::install();
@@ -499,12 +232,15 @@ mod tests {
#[test]
fn distance_weighs_letter_groups_half() {
use crate::affix_suggest::{fold, weighted_distance};
let map: Vec<Vec<char>> = MAP.iter().map(|g| g.chars().collect()).collect();
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1);
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1);
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2);
assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6);
assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3);
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1);
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1);
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2);
assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6);
assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3);
}
#[test]
+210
View File
@@ -0,0 +1,210 @@
//! The installed Hunspell dictionary of a proofing language, in one shape for every language:
//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`].
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::affix::AffixDict;
/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own.
pub(crate) trait Dictionary: Default + Send + Sync {
/// The installed `.aff` and `.dic` files, read.
fn load(aff: &Path, dic: &Path) -> Result<Self, String>;
/// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed.
fn parse(aff: &str, dic: &str) -> Result<Self, String>;
/// No words: the language has no spelling marks.
fn is_empty(&self) -> bool;
}
impl Dictionary for AffixDict {
fn load(aff: &Path, dic: &Path) -> Result<Self, String> {
crate::hunspell::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
AffixDict::from_hunspell(aff, dic, &[])
}
fn is_empty(&self) -> bool {
AffixDict::is_empty(self)
}
}
/// What every language's slot answers, whatever its dictionary type (the language table in
/// lib.rs holds them side by side).
pub(crate) trait Slot: Sync {
/// See [`DictSlot::name`].
fn name(&self) -> &'static str;
/// See [`DictSlot::installed_path`].
fn installed_path(&self) -> Option<PathBuf>;
/// See [`DictSlot::has_dictionary`].
fn has_dictionary(&self) -> bool;
/// See [`DictSlot::reload`].
fn reload(&self);
}
impl<D: Dictionary> Slot for DictSlot<D> {
fn name(&self) -> &'static str {
DictSlot::name(self)
}
fn installed_path(&self) -> Option<PathBuf> {
DictSlot::installed_path(self)
}
fn has_dictionary(&self) -> bool {
DictSlot::has_dictionary(self)
}
fn reload(&self) {
DictSlot::reload(self);
}
}
/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on
/// disk (it then has no file to report, even if a matching file also happens to exist): one lock,
/// so a reload and an install can't interleave their two fields.
struct State<D> {
dict: Option<Arc<D>>,
in_memory: bool,
/// Counts reloads, so a load that started before one is not kept.
reloads: u64,
}
/// One language's dictionary.
pub(crate) struct DictSlot<D> {
/// File names the dictionary may have, the usual one first (also the key of its load error).
names: &'static [&'static str],
/// The language's English name, for the log.
english: &'static str,
state: RwLock<State<D>>,
}
impl<D: Dictionary> DictSlot<D> {
pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self {
DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) }
}
fn read(&self) -> RwLockReadGuard<'_, State<D>> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State<D>> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
/// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names.
pub(crate) fn name(&self) -> &'static str {
self.names.first().copied().unwrap_or_default()
}
/// The file names the dictionary may have, the usual one first.
#[cfg(test)]
pub(crate) fn names(&self) -> &'static [&'static str] {
self.names
}
/// The dictionary: the installed one (read on first use), one given with [`Self::install`], or
/// an empty one when none is available (the language's spelling is off then).
pub(crate) fn get(&self) -> Arc<D> {
let reloads = {
let s = self.read();
if let Some(d) = s.dict.as_ref() {
return d.clone();
}
s.reloads
};
self.keep(reloads, Arc::new(self.load_installed()))
}
/// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the
/// load may have read the old folder: it serves this one check and the next one loads again.
fn keep(&self, reloads: u64, loaded: Arc<D>) -> Arc<D> {
let mut s = self.write();
if s.reloads != reloads {
return s.dict.clone().unwrap_or(loaded);
}
// Another thread (or an install) may have filled it meanwhile: that one wins.
s.dict.get_or_insert(loaded).clone()
}
fn load_installed(&self) -> D {
let name = self.name();
let Some((aff, dic)) = crate::hunspell::find(self.names) else {
crate::hunspell::clear_load_error(name);
log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english);
return D::default();
};
crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("{} spelling: {e}", self.english);
D::default()
})
}
/// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for
/// example one the user picked. On error the current dictionary stays.
pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> {
let d = D::parse(aff, dic)?;
let mut slot = self.write();
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub(crate) fn reload(&self) {
let mut slot = self.write();
slot.dict = None;
slot.in_memory = false;
slot.reloads = slot.reloads.wrapping_add(1);
}
/// The `.dic` file of the installed dictionary, searching the dictionaries folder and the
/// system locations now (File › Options › Proofing shows it); `None` when there is none, or
/// when the dictionary in use was given with [`Self::install`] and so has no file.
pub(crate) fn installed_path(&self) -> Option<PathBuf> {
if self.read().in_memory {
return None;
}
crate::hunspell::find(self.names).map(|(_, dic)| dic)
}
/// Is a dictionary available (spelling is checked)?
pub(crate) fn has_dictionary(&self) -> bool {
!self.get().is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
static SLOT: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_TEST"], "Test");
#[test]
fn an_installed_dictionary_has_no_file_until_reloaded() {
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
SLOT.install("", "1\nkot\n").unwrap();
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
assert_eq!(SLOT.installed_path(), None);
SLOT.reload();
assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none");
assert_eq!(SLOT.name(), "zz_SLOT_TEST");
}
/// A load that started before a dictionaries-folder change is not kept after its reload.
#[test]
fn a_load_from_before_a_reload_is_not_kept() {
let slot: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test");
let before = slot.read().reloads;
slot.reload();
let _ = slot.keep(before, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_none(), "the next check loads from the new folder");
let now = slot.read().reloads;
let _ = slot.keep(now, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_some(), "a load since the reload is kept");
}
}
+52
View File
@@ -0,0 +1,52 @@
//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian,
//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the
//! File › Options › Proofing choices. Each language goes on from there with its own rules.
use crate::Options;
use crate::affix::AffixDict;
/// Is `c` a letter of the Latin script (the only one these dictionaries cover)?
fn latin(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it
/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single
/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the
/// options skip those, or it is in another script.
pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> {
if d.is_empty() {
return None;
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return None;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return None;
}
w.chars().any(latin).then_some(w)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn words_needing_no_lookup() {
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
let defaults = Options::default();
assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary");
for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] {
assert_eq!(word_to_check(&d, w, &defaults), None, "{w}");
}
assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot"));
assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot"));
let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults };
assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO"));
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
}
}
+14
View File
@@ -26,6 +26,20 @@ pub struct TrDict {
max_suffix_len: usize,
}
impl crate::slot::Dictionary for TrDict {
fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
super::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
TrDict::from_hunspell(aff, dic)
}
fn is_empty(&self) -> bool {
TrDict::is_empty(self)
}
}
impl TrDict {
/// Returns true if empty (no dictionary loaded).
pub fn is_empty(&self) -> bool {
+43 -46
View File
@@ -15,39 +15,26 @@ pub mod dict;
pub mod grammar;
pub mod hyphen;
pub mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::slot::DictSlot;
pub use dict::TrDict;
pub use suggest::suggest;
/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<TrDict>>> = RwLock::new(None);
/// File names of a Turkish Hunspell dictionary.
const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"];
/// The Turkish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<TrDict> = DictSlot::new(NAMES, "Turkish");
/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available.
pub fn dictionary() -> Arc<TrDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> TrDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed");
return TrDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Turkish spelling: {e}");
TrDict::default()
})
DICT.get()
}
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
@@ -57,28 +44,15 @@ pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDic
TrDict::from_hunspell(&aff_text, &dic_text)
}
/// Use this Turkish dictionary from now on.
/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
/// current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = TrDict::from_hunspell(aff, dic)?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Reload the dictionary on next check.
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Turkish dictionary available?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Turkish?
@@ -86,15 +60,26 @@ pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain
/// numbers off, words with digits are checked against the dictionary too.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary.
pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // No Turkish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return true;
}
// Check user dictionary first
@@ -119,7 +104,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
}
// Compound words separated by hyphens (e.g., `afyon-karahisar`)
if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) {
if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) {
return true;
}
@@ -145,11 +130,8 @@ mod tests {
}
#[test]
fn installed_dictionary_checks_words_if_present() {
// Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check.
if !has_dictionary() {
return;
}
fn installed_dictionary_checks_words() {
test_dict::install();
assert!(is_correct("kitap"));
assert!(is_correct("kitaplar"));
assert!(is_correct("evde"));
@@ -164,4 +146,19 @@ mod tests {
crate::add_word("benimbilmedigimkelime");
assert!(is_correct("benimbilmedigimkelime"));
}
/// With Ignore words that contain numbers off, Turkish checks words with digits against the
/// dictionary instead of accepting them, hyphen parts included.
#[test]
fn proofing_options_reach_the_turkish_check() {
test_dict::install();
let lang = crate::ProofLang::Tr;
let defaults = crate::Options::default();
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("kitapp2", lang, &defaults));
assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits));
assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
}
}
+1 -4
View File
@@ -318,10 +318,7 @@ mod tests {
#[test]
fn typo_transposition_is_suggested() {
// Suggestions come from the installed Turkish dictionary; without one there are none.
if !super::super::has_dictionary() {
return;
}
crate::tr::test_dict::install();
let sugs = suggest("sorna", 3);
assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}");
+22
View File
@@ -0,0 +1,22 @@
//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
/// No affix rules: the forms the tests look up are listed whole.
pub const AFF: &str = "SET UTF-8\n";
/// The words the Turkish tests expect to be known.
pub const DIC: &str = "7
kitap
kitaplar
ev
evde
olacaklardan
sonra
geldin
";
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}