mirror of
https://github.com/storytold/wordcraft.git
synced 2026-11-03 00:40:33 -06:00
Merge pull request #5000 from benandez/integration/batch-proof
Integration batch proof: 2 approved proofing PRs (merge commit, please)
This commit is contained in:
@@ -8472,9 +8472,18 @@ fn next_footnote_skips_endnotes() {
|
||||
assert_eq!(s.sel.focus.off, 9);
|
||||
}
|
||||
|
||||
/// Tests that install a test dictionary wait for the dictionaries-folder test, which changes the
|
||||
/// folder and then reads what each language reports: an install in between reports no file.
|
||||
static DICTIONARIES: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
fn dictionaries_turn() -> std::sync::MutexGuard<'static, ()> {
|
||||
DICTIONARIES.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// A tiny synthetic Polish Hunspell dictionary (written for these tests). Real dictionaries are
|
||||
/// installed on the system, never bundled; every Polish test installs this same one.
|
||||
fn polish_test_dictionary() {
|
||||
let _turn = dictionaries_turn();
|
||||
wordcraft_proof::pl::install_dictionary(POLISH_TEST_AFF, POLISH_TEST_DIC).unwrap();
|
||||
}
|
||||
|
||||
@@ -8484,6 +8493,7 @@ const POLISH_TEST_DIC: &str = "12\npłynie\nrzeka\na\nto\nwiem\nże\ntak\nw\nog
|
||||
/// A tiny synthetic Portuguese Hunspell dictionary (written for these tests). Real dictionaries
|
||||
/// are installed on the system, never bundled; every Portuguese test installs this same one.
|
||||
fn portuguese_test_dictionary() {
|
||||
let _turn = dictionaries_turn();
|
||||
wordcraft_proof::pt::install_dictionary(PORTUGUESE_TEST_AFF, PORTUGUESE_TEST_DIC).unwrap();
|
||||
}
|
||||
|
||||
@@ -8497,6 +8507,7 @@ const PORTUGUESE_TEST_DIC: &str = "11\nsaudade\ncoração\ne\nele\ndisse\nnão\n
|
||||
/// running meanwhile see the same words; it stays set for the rest of the run for that reason.
|
||||
#[test]
|
||||
fn dictionaries_folder_option_is_saved_searched_and_reported() {
|
||||
let _turn = dictionaries_turn();
|
||||
let dir = std::env::temp_dir().join(format!("wc-engine-dictionaries-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
for (name, aff, dic) in [
|
||||
@@ -8527,14 +8538,20 @@ fn dictionaries_folder_option_is_saved_searched_and_reported() {
|
||||
assert_eq!(r["dictionariesFolder"], json!(dir.to_string_lossy()));
|
||||
let dicts = r["dictionaries"].as_array().unwrap();
|
||||
let file = |lang: &str| dicts.iter().find(|d| d["language"] == lang).map(|d| d["file"].clone());
|
||||
assert_eq!(file("pl"), Some(json!(dir.join("pl_PL.dic").to_string_lossy())));
|
||||
assert_eq!(file("pt"), Some(json!(dir.join("pt_BR.dic").to_string_lossy())));
|
||||
assert_eq!(file("id"), Some(json!(dir.join("id_ID.dic").to_string_lossy())));
|
||||
// Each language reports the folder's file, which is searched first.
|
||||
let reported = |got: Option<&serde_json::Value>, name: &str| {
|
||||
let want = dir.join(format!("{name}.dic"));
|
||||
assert_eq!(got, Some(&json!(want.to_string_lossy())), "{name}: {dicts:?}");
|
||||
assert_eq!(wordcraft_proof::hunspell::find(&[name]).map(|(_, dic)| dic), Some(want));
|
||||
};
|
||||
reported(file("pl").as_ref(), "pl_PL");
|
||||
reported(file("pt").as_ref(), "pt_BR");
|
||||
reported(file("id").as_ref(), "id_ID");
|
||||
// Each Spanish variety reports its own file (Spain, Mexico, Latin America, Uruguay, in order).
|
||||
let spanish: Vec<serde_json::Value> = dicts.iter().filter(|d| d["language"] == "es").map(|d| d["file"].clone()).collect();
|
||||
assert_eq!(spanish.len(), 4);
|
||||
assert_eq!(spanish.first(), Some(&json!(dir.join("es_ES.dic").to_string_lossy())));
|
||||
assert_eq!(spanish.last(), Some(&json!(dir.join("es_UY.dic").to_string_lossy())));
|
||||
reported(spanish.first(), "es_ES");
|
||||
reported(spanish.last(), "es_UY");
|
||||
// Polish text is checked against the folder's dictionary.
|
||||
run(&mut s, "document.setText", json!({"text": "rzeka płynie żeka"}));
|
||||
run(&mut s, "select.all", json!({}));
|
||||
@@ -8584,6 +8601,7 @@ fn portuguese_proofing_through_the_review_commands() {
|
||||
}
|
||||
|
||||
fn indonesian_test_dictionary() {
|
||||
let _turn = dictionaries_turn();
|
||||
wordcraft_proof::id::install_dictionary(INDONESIAN_TEST_AFF, INDONESIAN_TEST_DIC).unwrap();
|
||||
}
|
||||
|
||||
@@ -8755,6 +8773,7 @@ const URUGUAYAN_TEST_DIC: &str = "10\nesto\nes\nuna\nprueba/S\ndel\ncorrector\nt
|
||||
|
||||
fn spanish_test_dictionaries() {
|
||||
use wordcraft_proof::ProofLang;
|
||||
let _turn = dictionaries_turn();
|
||||
for (lang, dic) in [(ProofLang::Es, SPANISH_TEST_DIC), (ProofLang::EsUy, URUGUAYAN_TEST_DIC)] {
|
||||
wordcraft_proof::es::install_dictionary(lang, SPANISH_TEST_AFF, dic).unwrap();
|
||||
}
|
||||
@@ -8883,6 +8902,7 @@ const ITALIAN_TEST_AFF: &str = "SET UTF-8\nTRY aeiortnsclcaffèéè\nMAP 2\nMAP
|
||||
const ITALIAN_TEST_DIC: &str = "6\ncasa\ncaffè\nroma\ndisse\ne\nper\n";
|
||||
|
||||
fn italian_test_dictionary() {
|
||||
let _turn = dictionaries_turn();
|
||||
wordcraft_proof::it::install_dictionary(ITALIAN_TEST_AFF, ITALIAN_TEST_DIC).unwrap();
|
||||
}
|
||||
|
||||
|
||||
+501
-109
@@ -11,10 +11,14 @@
|
||||
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
|
||||
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
|
||||
//!
|
||||
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
|
||||
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
|
||||
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
|
||||
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
|
||||
//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`),
|
||||
//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and
|
||||
//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes
|
||||
//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the
|
||||
//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi`
|
||||
//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP`
|
||||
//! and `REP`.
|
||||
//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding,
|
||||
//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct.
|
||||
//!
|
||||
//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and
|
||||
@@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX;
|
||||
|
||||
/// Directives that change which words are correct in ways this engine doesn't implement.
|
||||
const UNSUPPORTED: &[&str] = &[
|
||||
"AF",
|
||||
"AM",
|
||||
"CHECKSHARPS",
|
||||
"CIRCUMFIX",
|
||||
"COMPLEXPREFIXES",
|
||||
"COMPOUNDBEGIN",
|
||||
"COMPOUNDEND",
|
||||
"COMPOUNDFIRST",
|
||||
"COMPOUNDFLAG",
|
||||
"COMPOUNDLAST",
|
||||
"COMPOUNDMIDDLE",
|
||||
"COMPOUNDMIN",
|
||||
"COMPOUNDPERMITFLAG",
|
||||
"COMPOUNDRULE",
|
||||
"FULLSTRIP",
|
||||
"ICONV",
|
||||
"IGNORE",
|
||||
"KEEPCASE",
|
||||
"LEMMA_PRESENT",
|
||||
"NEEDAFFIX",
|
||||
"OCONV",
|
||||
"ONLYINCOMPOUND",
|
||||
"PSEUDOROOT",
|
||||
"SUBSTANDARD",
|
||||
];
|
||||
|
||||
/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary
|
||||
/// uses 158).
|
||||
const MAX_FLAGS: usize = 256;
|
||||
|
||||
/// A set of flags, one bit per distinct flag character of the dictionary.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
|
||||
struct Mask([u64; MAX_FLAGS / 64]);
|
||||
|
||||
impl Mask {
|
||||
const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]);
|
||||
|
||||
/// The set holding flag number `i` alone (empty when `i` is out of range).
|
||||
fn bit(i: usize) -> Mask {
|
||||
let mut m = Mask::EMPTY;
|
||||
if let Some(w) = m.0.get_mut(i / 64) {
|
||||
*w = 1 << (i % 64);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// The number of the lowest flag in the set, if any.
|
||||
fn first(self) -> Option<usize> {
|
||||
self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize)
|
||||
}
|
||||
|
||||
fn is_empty(self) -> bool {
|
||||
self.0.iter().all(|&w| w == 0)
|
||||
}
|
||||
|
||||
/// Do the two sets share a flag?
|
||||
fn meets(self, other: Mask) -> bool {
|
||||
!(self & other).is_empty()
|
||||
}
|
||||
|
||||
/// Does this set hold every flag of `other`?
|
||||
fn covers(self, other: Mask) -> bool {
|
||||
(self & other) == other
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::BitAnd for Mask {
|
||||
type Output = Mask;
|
||||
fn bitand(mut self, o: Mask) -> Mask {
|
||||
for (a, b) in self.0.iter_mut().zip(o.0) {
|
||||
*a &= b;
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::BitOr for Mask {
|
||||
type Output = Mask;
|
||||
fn bitor(mut self, o: Mask) -> Mask {
|
||||
self |= o;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::BitOrAssign for Mask {
|
||||
fn bitor_assign(&mut self, o: Mask) {
|
||||
for (a, b) in self.0.iter_mut().zip(o.0) {
|
||||
*a |= b;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One character class of a rule's condition.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
enum Cond {
|
||||
@@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18;
|
||||
#[derive(Clone, Debug)]
|
||||
struct Rule {
|
||||
/// Bit of the rule's flag in the stems' flag masks.
|
||||
bit: u128,
|
||||
bit: Mask,
|
||||
flag: char,
|
||||
cross: bool,
|
||||
strip: Box<str>,
|
||||
add: Box<str>,
|
||||
/// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing
|
||||
/// further affixes. Chained application is bounded by [`MAX_CHAIN`].
|
||||
cont: u128,
|
||||
cont: Mask,
|
||||
cond: Box<[Cond]>,
|
||||
cond_text: Box<str>,
|
||||
}
|
||||
@@ -174,11 +239,13 @@ pub struct AffixDict {
|
||||
/// Per stem: index into `sets`.
|
||||
flags: Vec<u16>,
|
||||
/// Distinct flag masks.
|
||||
sets: Vec<u128>,
|
||||
sets: Vec<Mask>,
|
||||
/// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared).
|
||||
forbidden: u128,
|
||||
forbidden: Mask,
|
||||
/// Stems with NOSUGGEST are correct, but none of their forms are suggested.
|
||||
no_suggest: u128,
|
||||
no_suggest: Mask,
|
||||
/// Stems with SUBSTANDARD are correct too, and likewise never suggested.
|
||||
substandard: Mask,
|
||||
/// Open-addressing hash table of stem indices (`EMPTY` = free).
|
||||
table: Vec<u32>,
|
||||
/// Flag char of each bit.
|
||||
@@ -195,6 +262,12 @@ pub struct AffixDict {
|
||||
key: String,
|
||||
map: Vec<Vec<char>>,
|
||||
rep: Vec<(String, String)>,
|
||||
/// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none).
|
||||
keepcase: Mask,
|
||||
/// `ICONV`: input conversions applied before a lookup, `(from, to)`.
|
||||
iconv: Vec<(String, String)>,
|
||||
/// `IGNORE`: characters removed from input words (and from the stems and appended texts).
|
||||
ignore: Box<[char]>,
|
||||
/// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
|
||||
comments: Vec<String>,
|
||||
}
|
||||
@@ -263,6 +336,57 @@ struct Rules {
|
||||
/// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct.
|
||||
forbidden: Option<char>,
|
||||
no_suggest: Option<char>,
|
||||
substandard: Option<char>,
|
||||
/// The `KEEPCASE` flag.
|
||||
keepcase: Option<char>,
|
||||
iconv: Vec<(String, String)>,
|
||||
ignore: Vec<char>,
|
||||
/// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.
|
||||
aliases: Vec<String>,
|
||||
/// The `AF` table opened with its count line, so flag fields are alias numbers. A table
|
||||
/// without one is rejected, as in Hunspell: its lines are ignored.
|
||||
aliases_on: bool,
|
||||
}
|
||||
|
||||
impl Rules {
|
||||
/// Flags as written in a `.dic` entry or a continuation class: the flag characters
|
||||
/// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads
|
||||
/// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags:
|
||||
/// the stem is still a word, the rule still applies, without the flags (Hunspell warns).
|
||||
fn flags<'a>(&'a self, written: &'a str) -> &'a str {
|
||||
if !self.aliases_on {
|
||||
return written;
|
||||
}
|
||||
let digits = written.bytes().take_while(u8::is_ascii_digit).count();
|
||||
written
|
||||
.get(..digits)
|
||||
.and_then(|d| d.parse::<usize>().ok())
|
||||
.and_then(|n| n.checked_sub(1))
|
||||
.and_then(|i| self.aliases.get(i))
|
||||
.map_or("", String::as_str)
|
||||
}
|
||||
|
||||
/// `w` without the `IGNORE` characters.
|
||||
fn strip_ignored(&self, w: &str) -> String {
|
||||
if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }
|
||||
}
|
||||
}
|
||||
|
||||
/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a
|
||||
/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag.
|
||||
fn split_fields(line: &str) -> impl Iterator<Item = &str> {
|
||||
line.split([' ', '\t']).filter(|f| !f.is_empty())
|
||||
}
|
||||
|
||||
/// The one flag character a directive (`FORBIDDENWORD w`) names.
|
||||
fn one_flag(arg: &str, dir: &str, n: usize) -> Result<char, String> {
|
||||
let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR);
|
||||
match (it.next(), it.next()) {
|
||||
// Only the field separators of `split_fields` can't be a flag: a flag byte read as
|
||||
// U+0085 or U+00A0 is one.
|
||||
(Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c),
|
||||
_ => Err(format!("line {}: {dir} needs one flag character", n + 1)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
|
||||
@@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
let mut cross: HashMap<(bool, char), bool> = HashMap::new();
|
||||
let mut map_header = false;
|
||||
let mut rep_header = false;
|
||||
let mut af_header = false;
|
||||
let mut iconv_header = false;
|
||||
for (n, line) in aff.lines().enumerate() {
|
||||
let f: Vec<&str> = line.split_whitespace().collect();
|
||||
let f: Vec<&str> = split_fields(line).collect();
|
||||
let Some(&dir) = f.first() else { continue };
|
||||
if dir.starts_with('#') {
|
||||
continue;
|
||||
@@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
match dir {
|
||||
"SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS"
|
||||
| "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}
|
||||
// Morphological aliases, and a flag that only matters for morphological generation:
|
||||
// the words it marks are correct.
|
||||
"AM" | "LEMMA_PRESENT" => {}
|
||||
"SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?),
|
||||
// The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
|
||||
// trailing comment (`AF 1559 # aliases`).
|
||||
"AF" => {
|
||||
if !af_header {
|
||||
af_header = true;
|
||||
// A count below 1 rejects the table too: flags stay literal (`AF 0`).
|
||||
r.aliases_on = at(1).parse::<usize>().is_ok_and(|n| n > 0);
|
||||
} else if r.aliases_on {
|
||||
r.aliases.push(at(1).to_string());
|
||||
}
|
||||
}
|
||||
"ICONV" => {
|
||||
if !iconv_header && at(1).parse::<usize>().is_ok() {
|
||||
iconv_header = true;
|
||||
} else if f.len() >= 3 {
|
||||
r.iconv.push((at(1).to_string(), at(2).to_string()));
|
||||
}
|
||||
}
|
||||
"IGNORE" => r.ignore = at(1).chars().collect(),
|
||||
"KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?),
|
||||
// Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
|
||||
// with the one-character parsing here; only the multi-character encodings differ.
|
||||
"FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => {
|
||||
return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
|
||||
}
|
||||
"FLAG" => {}
|
||||
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
|
||||
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
|
||||
match (it.next(), it.next()) {
|
||||
(Some(c), None) if !c.is_whitespace() => {
|
||||
if directive == "FORBIDDENWORD" {
|
||||
r.forbidden = Some(c);
|
||||
} else {
|
||||
r.no_suggest = Some(c);
|
||||
}
|
||||
}
|
||||
_ => return Err(format!("line {}: {directive} needs one flag character", n + 1)),
|
||||
}
|
||||
}
|
||||
"FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?),
|
||||
"NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?),
|
||||
"TRY" => r.try_chars = cap_try(at(1)),
|
||||
"KEY" => r.key = at(1).to_string(),
|
||||
"MAP" => {
|
||||
@@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
}
|
||||
}
|
||||
"REP" => {
|
||||
if !rep_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
|
||||
if !rep_header && at(1).parse::<usize>().is_ok() {
|
||||
rep_header = true;
|
||||
} else if f.len() >= 3 {
|
||||
r.rep.push((at(1).replace('_', " "), at(2).replace('_', " ")));
|
||||
@@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// Aliases may be declared after the rules that use them; resolve once all are read.
|
||||
// Hunspell removes `IGNORE` characters from the appended text only: a stripped text that
|
||||
// holds one never matches a stem (stems lose them too), so the rule never applies.
|
||||
let resolve = |rules: &mut Vec<RawRule>, r: &Rules| {
|
||||
for rule in rules.iter_mut() {
|
||||
rule.4 = r.flags(&rule.4).to_string();
|
||||
rule.3 = r.strip_ignored(&rule.3);
|
||||
}
|
||||
};
|
||||
let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));
|
||||
resolve(&mut pfx, &r);
|
||||
resolve(&mut sfx, &r);
|
||||
(r.prefixes, r.suffixes) = (pfx, sfx);
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
@@ -397,7 +549,12 @@ impl AffixDict {
|
||||
let rules = parse_aff(aff)?;
|
||||
let mut stems: Vec<(String, String)> = Vec::new();
|
||||
for (n, line) in dic.lines().enumerate() {
|
||||
let Some(tok) = line.split_whitespace().next() else { continue };
|
||||
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
|
||||
// Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).
|
||||
let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();
|
||||
if tok.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if n == 0 && tok.parse::<usize>().is_ok() {
|
||||
continue;
|
||||
}
|
||||
@@ -420,9 +577,11 @@ impl AffixDict {
|
||||
word.push(c);
|
||||
}
|
||||
}
|
||||
let word = rules.strip_ignored(&word);
|
||||
if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
|
||||
continue;
|
||||
}
|
||||
let flags = rules.flags(&flags).to_string();
|
||||
stems.push((word, flags));
|
||||
}
|
||||
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
|
||||
@@ -434,11 +593,12 @@ impl AffixDict {
|
||||
flag_chars.extend(stems.iter().flat_map(|s| s.1.chars()));
|
||||
flag_chars.sort_unstable();
|
||||
flag_chars.dedup();
|
||||
if flag_chars.len() > 128 {
|
||||
return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len()));
|
||||
if flag_chars.len() > MAX_FLAGS {
|
||||
return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len()));
|
||||
}
|
||||
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i);
|
||||
let forbidden = rules.forbidden.map_or(0, bit_of);
|
||||
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit);
|
||||
let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of);
|
||||
let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of);
|
||||
// Stems sorted by bytes; homonyms (the same word twice) are merged.
|
||||
stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
|
||||
let mut d = AffixDict {
|
||||
@@ -446,7 +606,11 @@ impl AffixDict {
|
||||
offs: Vec::with_capacity(stems.len() + 1),
|
||||
flag_chars: flag_chars.clone(),
|
||||
forbidden,
|
||||
no_suggest: rules.no_suggest.map_or(0, bit_of),
|
||||
no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of),
|
||||
substandard: rules.substandard.map_or(Mask::EMPTY, bit_of),
|
||||
keepcase,
|
||||
iconv: rules.iconv,
|
||||
ignore: rules.ignore.into_boxed_slice(),
|
||||
try_chars: rules.try_chars,
|
||||
key: rules.key,
|
||||
map: rules.map,
|
||||
@@ -455,16 +619,16 @@ impl AffixDict {
|
||||
..AffixDict::default()
|
||||
};
|
||||
d.offs.push(0);
|
||||
let mut set_ids: HashMap<u128, u16> = HashMap::new();
|
||||
let mut set_ids: HashMap<Mask, u16> = HashMap::new();
|
||||
let mut i = 0;
|
||||
while i < stems.len() {
|
||||
let Some((word, _)) = stems.get(i) else { break };
|
||||
let mut mask = 0u128;
|
||||
let mut mask = Mask::EMPTY;
|
||||
let mut j = i;
|
||||
while let Some((w, f)) = stems.get(j)
|
||||
&& w == word
|
||||
{
|
||||
mask |= f.chars().map(bit_of).fold(0, |a, b| a | b);
|
||||
mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b);
|
||||
j += 1;
|
||||
}
|
||||
let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?;
|
||||
@@ -478,7 +642,7 @@ impl AffixDict {
|
||||
i = j;
|
||||
}
|
||||
let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result<Rule, String> {
|
||||
let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c));
|
||||
let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c));
|
||||
Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() })
|
||||
};
|
||||
d.prefixes = rules.prefixes.into_iter().map(mk).collect::<Result<_, _>>()?;
|
||||
@@ -493,7 +657,7 @@ impl AffixDict {
|
||||
// Each (appended, stripped) pair's group, found by hash rather than by scanning the
|
||||
// appended string's groups: a dictionary with many strips per suffix stays linear.
|
||||
let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
|
||||
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
|
||||
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); MAX_FLAGS];
|
||||
for (i, r) in self.suffixes.iter().enumerate() {
|
||||
let groups = idx.entry(r.add.clone()).or_default();
|
||||
let Ok(i) = u32::try_from(i) else { break };
|
||||
@@ -504,9 +668,7 @@ impl AffixDict {
|
||||
groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
|
||||
}
|
||||
}
|
||||
if r.bit != 0
|
||||
&& let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
|
||||
{
|
||||
if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) {
|
||||
list.push(i);
|
||||
}
|
||||
}
|
||||
@@ -551,12 +713,12 @@ impl AffixDict {
|
||||
}
|
||||
|
||||
/// The flag characters of `mask`, in flag order (how the compact format stores flags).
|
||||
fn flag_chars_for(&self, mask: u128) -> String {
|
||||
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect()
|
||||
fn flag_chars_for(&self, mask: Mask) -> String {
|
||||
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect()
|
||||
}
|
||||
|
||||
/// The character of `mask` when it holds exactly one flag.
|
||||
fn flag_char(&self, mask: u128) -> Option<char> {
|
||||
fn flag_char(&self, mask: Mask) -> Option<char> {
|
||||
let s = self.flag_chars_for(mask);
|
||||
let mut it = s.chars();
|
||||
match (it.next(), it.next()) {
|
||||
@@ -589,11 +751,12 @@ impl AffixDict {
|
||||
}
|
||||
|
||||
/// Flag mask of a stem, if `w` is one.
|
||||
fn stem_flags(&self, w: &str) -> Option<u128> {
|
||||
self.stem_flags_excluding(w, self.forbidden)
|
||||
fn stem_flags(&self, w: &str) -> Option<Mask> {
|
||||
self.stem_flags_except(w, Mask::EMPTY)
|
||||
}
|
||||
|
||||
fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option<u128> {
|
||||
/// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`.
|
||||
fn stem_flags_except(&self, w: &str, deny: Mask) -> Option<Mask> {
|
||||
if self.table.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -607,8 +770,9 @@ impl AffixDict {
|
||||
if self.word(t as usize) == Some(w) {
|
||||
let id = *self.flags.get(t as usize)?;
|
||||
let mask = self.sets.get(usize::from(id)).copied();
|
||||
// All derivations use this lookup: exclusions apply to stems and their forms.
|
||||
return mask.filter(|m| m & excluded == 0);
|
||||
// A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit,
|
||||
// and (this is the only stem lookup) no affixed form is built on it either.
|
||||
return mask.filter(|m| !m.meets(self.forbidden | deny));
|
||||
}
|
||||
slot = (slot + 1) & mask;
|
||||
}
|
||||
@@ -621,8 +785,8 @@ impl AffixDict {
|
||||
/// The rules of a group that could lead to `w` are merged into one mask of their flags and
|
||||
/// followed once, not once per rule: the answer is the same, but thousands of rules of one
|
||||
/// flag cost one walk, not one each (#3613).
|
||||
fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
|
||||
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
|
||||
fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool {
|
||||
if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) {
|
||||
return true;
|
||||
}
|
||||
if depth == 0 {
|
||||
@@ -641,9 +805,9 @@ impl AffixDict {
|
||||
.rules
|
||||
.iter()
|
||||
.filter_map(|&i| self.suffixes.get(i as usize))
|
||||
.filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
|
||||
.fold(0u128, |m, r| m | r.bit);
|
||||
if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
|
||||
.filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true))
|
||||
.fold(Mask::EMPTY, |m, r| m | r.bit);
|
||||
if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -651,21 +815,55 @@ impl AffixDict {
|
||||
false
|
||||
}
|
||||
|
||||
/// Is `w` exactly (case and all) a stem or an affixed form of one?
|
||||
/// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE`
|
||||
/// apply first.
|
||||
pub fn contains(&self, w: &str) -> bool {
|
||||
self.contains_excluding(w, self.forbidden)
|
||||
self.contains_except(&self.normalize(w), Mask::EMPTY)
|
||||
}
|
||||
|
||||
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded).
|
||||
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD
|
||||
/// stems excluded).
|
||||
pub fn contains_suggestible(&self, w: &str) -> bool {
|
||||
self.contains_excluding(w, self.forbidden | self.no_suggest)
|
||||
self.contains_except(&self.normalize(w), self.no_suggest | self.substandard)
|
||||
}
|
||||
|
||||
fn contains_excluding(&self, w: &str, excluded: u128) -> bool {
|
||||
/// `w` after the `ICONV` conversions (at each position the longest match wins) and without
|
||||
/// the `IGNORE` characters.
|
||||
fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
|
||||
if self.iconv.is_empty() && self.ignore.is_empty() {
|
||||
return std::borrow::Cow::Borrowed(w);
|
||||
}
|
||||
// One pass: the input is copied only from the first change on.
|
||||
let mut out: Option<String> = None;
|
||||
let mut rest = w;
|
||||
while let Some(c) = rest.chars().next() {
|
||||
let done = w.get(..w.len() - rest.len()).unwrap_or_default();
|
||||
let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
|
||||
match hit {
|
||||
Some((from, to)) => {
|
||||
out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));
|
||||
rest = rest.get(from.len()..).unwrap_or_default();
|
||||
}
|
||||
None => {
|
||||
if self.ignore.contains(&c) {
|
||||
out.get_or_insert_with(|| done.to_string());
|
||||
} else if let Some(o) = out.as_mut() {
|
||||
o.push(c);
|
||||
}
|
||||
rest = rest.get(c.len_utf8()..).unwrap_or_default();
|
||||
}
|
||||
}
|
||||
}
|
||||
out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
|
||||
}
|
||||
|
||||
/// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't
|
||||
/// count (case variants of `KEEPCASE` stems).
|
||||
fn contains_except(&self, w: &str, deny: Mask) -> bool {
|
||||
if w.is_empty() || w.len() > MAX_WORD_BYTES {
|
||||
return false;
|
||||
}
|
||||
if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) {
|
||||
if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) {
|
||||
return true;
|
||||
}
|
||||
let mut root = String::with_capacity(w.len() + 8);
|
||||
@@ -680,13 +878,13 @@ impl AffixDict {
|
||||
if !r.cond_matches(&root, false) {
|
||||
continue;
|
||||
}
|
||||
if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) {
|
||||
if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) {
|
||||
return true;
|
||||
}
|
||||
if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) {
|
||||
if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) {
|
||||
return true;
|
||||
}
|
||||
if r.cross && self.suffixed(&root, r.bit, excluded) {
|
||||
if r.cross && self.suffixed(&root, r.bit, deny) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -695,7 +893,7 @@ impl AffixDict {
|
||||
|
||||
/// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must
|
||||
/// carry those flags too and the suffix must allow cross products.
|
||||
fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool {
|
||||
fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool {
|
||||
let mut root = String::with_capacity(w.len() + 8);
|
||||
// Each split point from the end: the appended string is `w[b..]`, at most the longest
|
||||
// suffix, and something must stay before it.
|
||||
@@ -710,18 +908,18 @@ impl AffixDict {
|
||||
let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize));
|
||||
// The stem path: `root` is a stem carrying the rule's flag (and the prefix's,
|
||||
// when a cross product is being checked).
|
||||
if let Some(f) = self.stem_flags_excluding(&root, excluded)
|
||||
&& f & need == need
|
||||
&& rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true))
|
||||
if let Some(f) = self.stem_flags_except(&root, deny)
|
||||
&& f.covers(need)
|
||||
&& rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// Continuation classes: `root` need not be a stem carrying this rule's flag —
|
||||
// it may be a form that gained the flag from another rule's continuation. The
|
||||
// matching rules' flags are looked for together, in one walk.
|
||||
if need == 0 {
|
||||
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
|
||||
if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
|
||||
if need.is_empty() {
|
||||
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit);
|
||||
if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -732,26 +930,35 @@ impl AffixDict {
|
||||
|
||||
/// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals
|
||||
/// — in lower case (and capitalized), since a sentence may start with any word and headings
|
||||
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`).
|
||||
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a
|
||||
/// `KEEPCASE` stem matches only as written.
|
||||
pub fn check(&self, w: &str) -> bool {
|
||||
self.check_excluding(w, self.forbidden)
|
||||
self.check_except(w, Mask::EMPTY)
|
||||
}
|
||||
|
||||
/// Whether a suggestion is correct in its displayed case and has an eligible stem.
|
||||
pub fn can_suggest(&self, w: &str) -> bool {
|
||||
self.check_excluding(w, self.forbidden | self.no_suggest)
|
||||
self.check_except(w, self.no_suggest | self.substandard)
|
||||
}
|
||||
|
||||
fn check_excluding(&self, w: &str, excluded: u128) -> bool {
|
||||
if self.contains_excluding(w, excluded) {
|
||||
/// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case
|
||||
/// alone (`km` from `km/K`, which `Km` and `KM` are not).
|
||||
pub(crate) fn case_locked(&self, w: &str) -> bool {
|
||||
self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase)
|
||||
}
|
||||
|
||||
fn check_except(&self, w: &str, deny: Mask) -> bool {
|
||||
let w = self.normalize(w);
|
||||
if self.contains_except(&w, deny) {
|
||||
return true;
|
||||
}
|
||||
match casing(w) {
|
||||
let deny = deny | self.keepcase;
|
||||
match casing(&w) {
|
||||
Casing::Lower | Casing::Mixed => false,
|
||||
Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded),
|
||||
Casing::Initial => self.contains_except(&w.to_lowercase(), deny),
|
||||
Casing::Upper => {
|
||||
let lower = w.to_lowercase();
|
||||
self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded)
|
||||
self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -766,11 +973,11 @@ impl AffixDict {
|
||||
let mut cross_sfx = Vec::new();
|
||||
// Suffix forms queued with the continuation classes they carry, for chained expansion;
|
||||
// each (form, classes) pair is expanded once.
|
||||
let mut chain: Vec<(String, u128)> = Vec::new();
|
||||
let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
|
||||
let mut chain: Vec<(String, Mask)> = Vec::new();
|
||||
let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new();
|
||||
let mut work = 0usize;
|
||||
for r in &self.suffixes {
|
||||
if f & r.bit == 0 || !r.cond_matches(stem, true) {
|
||||
if !f.meets(r.bit) || !r.cond_matches(stem, true) {
|
||||
continue;
|
||||
}
|
||||
let Some(base) = stem.strip_suffix(&*r.strip) else { continue };
|
||||
@@ -778,7 +985,7 @@ impl AffixDict {
|
||||
if r.cross {
|
||||
cross_sfx.push(form.clone());
|
||||
}
|
||||
if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
|
||||
if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) {
|
||||
chain.push((form.clone(), r.cont));
|
||||
}
|
||||
out.push(form);
|
||||
@@ -804,7 +1011,7 @@ impl AffixDict {
|
||||
}
|
||||
let Some(base) = form.strip_suffix(&*r.strip) else { continue };
|
||||
let next = format!("{base}{}", r.add);
|
||||
if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
|
||||
if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) {
|
||||
chain.push((next.clone(), r.cont));
|
||||
}
|
||||
out.push(next);
|
||||
@@ -815,7 +1022,7 @@ impl AffixDict {
|
||||
}
|
||||
}
|
||||
for r in &self.prefixes {
|
||||
if f & r.bit == 0 || !r.cond_matches(stem, false) {
|
||||
if !f.meets(r.bit) || !r.cond_matches(stem, false) {
|
||||
continue;
|
||||
}
|
||||
let Some(rest) = stem.strip_prefix(&*r.strip) else { continue };
|
||||
@@ -839,11 +1046,11 @@ impl AffixDict {
|
||||
}
|
||||
|
||||
/// The suffix rules carrying any flag of `mask`, in rule order.
|
||||
fn suffixes_with(&self, mask: u128) -> Vec<u32> {
|
||||
fn suffixes_with(&self, mask: Mask) -> Vec<u32> {
|
||||
let mut rules: Vec<u32> = Vec::new();
|
||||
let mut lists = 0;
|
||||
for (bit, list) in self.sfx_by_flag.iter().enumerate() {
|
||||
if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
|
||||
if mask.meets(Mask::bit(bit)) && !list.is_empty() {
|
||||
rules.extend_from_slice(list);
|
||||
lists += 1;
|
||||
}
|
||||
@@ -904,13 +1111,21 @@ impl AffixDict {
|
||||
for (a, b) in &self.rep {
|
||||
out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_")));
|
||||
}
|
||||
for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] {
|
||||
if mask != 0
|
||||
for (directive, mask) in
|
||||
[("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)]
|
||||
{
|
||||
if !mask.is_empty()
|
||||
&& let Some(c) = self.flag_char(mask)
|
||||
{
|
||||
out.push_str(&format!("{directive} {c}\n"));
|
||||
}
|
||||
}
|
||||
for (a, b) in &self.iconv {
|
||||
out.push_str(&format!("ICONV {a} {b}\n"));
|
||||
}
|
||||
if !self.ignore.is_empty() {
|
||||
out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));
|
||||
}
|
||||
let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() };
|
||||
for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] {
|
||||
for r in rules {
|
||||
@@ -934,8 +1149,8 @@ impl AffixDict {
|
||||
let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255);
|
||||
raw.push(p as u8);
|
||||
raw.extend_from_slice(b.get(p..).unwrap_or_default());
|
||||
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0);
|
||||
if mask != 0 {
|
||||
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default();
|
||||
if !mask.is_empty() {
|
||||
raw.push(b'/');
|
||||
raw.extend_from_slice(self.flag_chars_for(mask).as_bytes());
|
||||
}
|
||||
@@ -962,26 +1177,24 @@ impl AffixDict {
|
||||
comments.push(c.to_string());
|
||||
continue;
|
||||
}
|
||||
let f: Vec<&str> = line.split_whitespace().collect();
|
||||
let f: Vec<&str> = split_fields(line).collect();
|
||||
let at = |k: usize| f.get(k).copied().unwrap_or("");
|
||||
match at(0) {
|
||||
"TRY" => rules.try_chars = cap_try(at(1)),
|
||||
"KEY" => rules.key = at(1).to_string(),
|
||||
"MAP" => rules.map.push(at(1).chars().collect()),
|
||||
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
|
||||
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
|
||||
let mut it = at(1).chars();
|
||||
match (it.next(), it.next()) {
|
||||
(Some(c), None) if !c.is_whitespace() => {
|
||||
if directive == "FORBIDDENWORD" {
|
||||
rules.forbidden = Some(c);
|
||||
} else {
|
||||
rules.no_suggest = Some(c);
|
||||
}
|
||||
}
|
||||
_ => return Err(format!("affix dictionary: bad {directive} in `{line}`")),
|
||||
directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => {
|
||||
let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?;
|
||||
match directive {
|
||||
"FORBIDDENWORD" => rules.forbidden = Some(c),
|
||||
"NOSUGGEST" => rules.no_suggest = Some(c),
|
||||
"SUBSTANDARD" => rules.substandard = Some(c),
|
||||
_ => rules.keepcase = Some(c),
|
||||
}
|
||||
}
|
||||
"ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),
|
||||
"IGNORE" => rules.ignore = at(1).chars().collect(),
|
||||
"MAP" => rules.map.push(at(1).chars().collect()),
|
||||
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
|
||||
kind @ ("PFX" | "SFX") => {
|
||||
let mut fc = at(1).chars();
|
||||
let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) };
|
||||
@@ -1223,7 +1436,7 @@ pub(crate) mod tests {
|
||||
#[test]
|
||||
fn more_than_64_distinct_flags_load() {
|
||||
// VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin
|
||||
// letter, the digits and eight accented letters: 70 flags, 64 < n <= 128.
|
||||
// letter, the digits and eight accented letters: 70 flags, more than 64.
|
||||
let mut flags = String::new();
|
||||
let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n");
|
||||
let chars: Vec<char> = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect();
|
||||
@@ -1239,6 +1452,24 @@ pub(crate) mod tests {
|
||||
assert!(!d.contains("verboß"), "ß is not a flag and not a stem");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_than_128_distinct_flags_load() {
|
||||
// The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up.
|
||||
let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect();
|
||||
let mut aff = String::from("SET UTF-8\n");
|
||||
for (i, c) in flags.chars().enumerate() {
|
||||
aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n"));
|
||||
}
|
||||
let half: String = flags.chars().step_by(2).collect();
|
||||
let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap();
|
||||
assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130"));
|
||||
assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131"));
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert!(back.contains("szóx198") && !back.contains("szóx199"));
|
||||
let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect();
|
||||
assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forbidden_words_are_rejected() {
|
||||
// VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying
|
||||
@@ -1364,6 +1595,167 @@ pub(crate) mod tests {
|
||||
assert!(d.contains("kot") && d.contains("koty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flag_aliases_stand_for_flag_sets() {
|
||||
// `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags.
|
||||
// Morphological aliases (`AM`) and the fields after a stem are skipped.
|
||||
let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n";
|
||||
let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();
|
||||
assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));
|
||||
assert!(!d.contains("házban"), "the stem has a and b, not c");
|
||||
// An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes.
|
||||
let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap();
|
||||
assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined");
|
||||
assert!(undefined.check("kertok"), "the other entries keep their aliases");
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iconv_and_ignore_normalize_input() {
|
||||
// Ligatures read as their letters; `IGNORE` characters vanish from input and stems.
|
||||
let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap();
|
||||
assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine"));
|
||||
assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)"));
|
||||
assert!(!d.check("finer"));
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alias_numbers_are_read_as_hunspell_reads_them() {
|
||||
// Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias
|
||||
// its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags.
|
||||
let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap();
|
||||
assert!(d.check("lab") && !d.check("labq") && !d.check("labs"));
|
||||
assert!(d.check("falx") && !d.check("fals") && d.check("kuts"));
|
||||
// A table without its count line is rejected: no alias applies.
|
||||
let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap();
|
||||
assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals"));
|
||||
// `AF 0` is rejected the same way: `lab/B` keeps its literal flag.
|
||||
let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap();
|
||||
assert!(zero.check("lab") && zero.check("labs"));
|
||||
// An undefined alias in a continuation class leaves the rule in force, without it.
|
||||
let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap();
|
||||
assert!(cont.check("haz") && cont.check("hazok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rep_count_line_may_carry_a_comment() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap();
|
||||
assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_lines_may_carry_a_comment() {
|
||||
// Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`
|
||||
// is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`.
|
||||
let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap();
|
||||
assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`");
|
||||
assert!(d.check("a1") && !d.check("a#"));
|
||||
// The UTF-8 reader keeps such a line as it is, so both readers agree on it.
|
||||
let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap();
|
||||
assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}");
|
||||
assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dic_entries_may_hold_spaces() {
|
||||
// `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.
|
||||
let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";
|
||||
// Checked with the Hunspell 1.7 command line: without a tab or a morphological field the
|
||||
// whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it
|
||||
// (` lab` is no `lab`).
|
||||
let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap();
|
||||
assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys"));
|
||||
assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így"));
|
||||
assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));
|
||||
assert!(!d.contains("lab") && !d.contains("labs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignored_characters_leave_the_stripped_text_alone() {
|
||||
// Hunspell removes `IGNORE` characters from the appended text, not from the stripped
|
||||
// one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong,
|
||||
// while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line).
|
||||
let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap();
|
||||
assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac"));
|
||||
let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap();
|
||||
assert!(add.check("ac") && add.check("acx"));
|
||||
for d in [&strip, &add] {
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert_eq!(back.check("ac"), d.check("ac"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizing_borrows_when_nothing_changes() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();
|
||||
assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine")));
|
||||
assert_eq!(d.normalize("(fi)ne"), "fine");
|
||||
assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keepcase_stems_match_only_as_written() {
|
||||
let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap();
|
||||
assert!(d.check("pH") && d.check("km") && d.check("kms"));
|
||||
assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms"));
|
||||
assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case");
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert!(back.check("km") && !back.check("Km"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn substandard_and_lemma_present_words_are_correct() {
|
||||
// `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD
|
||||
// stem is never suggested; the stems and rules they mark are still correct.
|
||||
let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap();
|
||||
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
|
||||
assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem"));
|
||||
assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem"));
|
||||
assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot"));
|
||||
assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_directives_are_refused_and_the_rest_load() {
|
||||
// A dictionary that needs a directive outside this engine fails to load rather than
|
||||
// accepting the wrong words.
|
||||
for dir in UNSUPPORTED {
|
||||
assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused");
|
||||
}
|
||||
// The directives this engine reads, with no other change to the dictionary (`AF` would
|
||||
// turn the flags of `DIC` into alias numbers; it has tests of its own).
|
||||
for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {
|
||||
let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}"));
|
||||
assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fields_split_on_spaces_and_tabs_only() {
|
||||
// A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator.
|
||||
let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap();
|
||||
assert!(d.contains("koty") && d.contains("kotem"));
|
||||
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
|
||||
assert!(back.contains("koty") && back.contains("kotem"));
|
||||
// The same bytes as one-flag directives, as Hunspell reads them.
|
||||
let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap();
|
||||
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
|
||||
assert!(dict.check("kots") && !dict.check("rossz"));
|
||||
assert!(dict.check("pH") && !dict.check("PH"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_and_broken_input_is_an_error() {
|
||||
assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err());
|
||||
|
||||
@@ -0,0 +1,556 @@
|
||||
//! Spelling suggestions from an affix dictionary, one engine for every language that reads one
|
||||
//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come
|
||||
//! in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`warszawa` → `Warszawa`);
|
||||
//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`):
|
||||
//! adding a missing accent is the commonest slip and ranks first, while taking away one the
|
||||
//! writer typed ranks after the known confusions;
|
||||
//! 3. the REP table of common confusions (`żeka` → `rzeka`);
|
||||
//! 4. a missing space (`napewno` → `na pewno`);
|
||||
//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with
|
||||
//! the next or mistyped (TRY letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in
|
||||
//! which a MAP slip costs half (two typos at most).
|
||||
//!
|
||||
//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may
|
||||
//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every
|
||||
//! round, so a call costs a few thousand dictionary lookups at most.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Longest word suggestions are looked for, in letters.
|
||||
pub(crate) const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
|
||||
pub(crate) const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
pub(crate) const COST_SPLIT: u32 = 12;
|
||||
/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]).
|
||||
const COST_SPLIT_SHORT: u32 = 17;
|
||||
/// A missing space between two longer halves with no short word beside it ([`Split::Near`]).
|
||||
pub(crate) const COST_SPLIT_LONG: u32 = 18;
|
||||
const COST_KEY: u32 = 13;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
pub(crate) const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
/// What a language adds to its dictionary's tables.
|
||||
pub(crate) struct Tuning {
|
||||
/// Where a missing space may go (round 4).
|
||||
pub split: Split,
|
||||
/// TRY letters for a dictionary without a TRY line.
|
||||
pub alphabet: &'static str,
|
||||
/// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`).
|
||||
pub map: &'static [&'static str],
|
||||
/// REP confusions the language adds to its dictionary's.
|
||||
pub rep: &'static [(&'static str, &'static str)],
|
||||
/// The cost of a REP confusion.
|
||||
pub rep_cost: u32,
|
||||
/// Most letters a MAP variant changes.
|
||||
pub max_map_changes: usize,
|
||||
/// The cost of changing letter `from` to its MAP relative `to`.
|
||||
pub map_cost: fn(char, char) -> u32,
|
||||
/// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead.
|
||||
pub mistype_cost: fn(&[char], usize, char) -> u32,
|
||||
}
|
||||
|
||||
/// Where a missing space may go.
|
||||
pub(crate) enum Split {
|
||||
/// Anywhere, dearer when a half is shorter than three letters.
|
||||
Anywhere,
|
||||
/// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one
|
||||
/// that runs into the word before (`kocham cię`), dearer between two longer halves, and never
|
||||
/// leaving a stray short half (`że ka`).
|
||||
Near { leading: &'static [&'static str], trailing: &'static [&'static str] },
|
||||
/// The language's own rule: the cost of the halves `a b`, or `None` to skip them.
|
||||
Custom(fn(&str, &str) -> Option<u32>),
|
||||
}
|
||||
|
||||
impl Tuning {
|
||||
/// The dictionary's tables alone (Czech, Indonesian, Spanish).
|
||||
pub(crate) const DEFAULT: Tuning =
|
||||
Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost };
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (typing without diacritics); an accent the writer did type is likelier right,
|
||||
/// so taking one away costs more than a known confusion from the REP table.
|
||||
pub(crate) fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Any mistyped letter costs the same.
|
||||
fn mistype_cost(_: &[char], _: usize, _: char) -> u32 {
|
||||
COST_MISTYPE
|
||||
}
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
/// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost).
|
||||
tested: HashMap<String, u32>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input {
|
||||
return;
|
||||
}
|
||||
match self.tested.get_mut(cand) {
|
||||
Some(prev) if *prev <= cost => return,
|
||||
Some(prev) => *prev = cost,
|
||||
None => {
|
||||
self.tested.insert(cand.to_string(), cost);
|
||||
}
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moze` → `może`, then
|
||||
// `Mozę`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP groups: the dictionary's, then the language's.
|
||||
fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec<char>]> {
|
||||
if t.map.is_empty() {
|
||||
return Cow::Borrowed(d.map());
|
||||
}
|
||||
Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect())
|
||||
}
|
||||
|
||||
/// The other members of every MAP group of `c` (its likely confusions), each once.
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::new();
|
||||
for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) {
|
||||
if x != c && !out.contains(&x) {
|
||||
out.push(x);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
|
||||
/// letter by letter), with its cost, each written after `buf`.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some((&c, rest)) = chars.split_first() else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
|
||||
buf.push(c);
|
||||
map_variants(rest, later, t, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < t.max_map_changes {
|
||||
for &a in here {
|
||||
buf.push(a);
|
||||
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`.
|
||||
pub(crate) fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered.
|
||||
fn split_cost(split: &Split, a: &str, b: &str) -> Option<u32> {
|
||||
let long = |part: &str| part.chars().count() >= 3;
|
||||
match split {
|
||||
Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }),
|
||||
Split::Near { leading, trailing } => {
|
||||
if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) {
|
||||
Some(COST_SPLIT)
|
||||
} else if long(a) && long(b) {
|
||||
Some(COST_SPLIT_LONG)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Split::Custom(cost) => cost(a, b),
|
||||
}
|
||||
}
|
||||
|
||||
/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case.
|
||||
pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
|
||||
add_case(&mut col, word, &input);
|
||||
let map = groups(d, t);
|
||||
add_map_slips(&mut col, &chars, &map, t);
|
||||
add_confusions(&mut col, &input, t);
|
||||
add_splits(&mut col, &input, &t.split);
|
||||
add_key_neighbours(&mut col, &chars);
|
||||
add_typos(&mut col, &chars, t);
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars, &map);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (the first changed letter breaks ties).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let recased = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text.clone(),
|
||||
};
|
||||
// A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`.
|
||||
let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased };
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 1: letter case.
|
||||
fn add_case(col: &mut Collector, word: &str, input: &str) {
|
||||
if col.d.contains(input) && word != input {
|
||||
col.keep(input.to_string(), COST_CASE);
|
||||
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
|
||||
col.keep(capitalize(input), COST_CASE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 2: accents and other MAP slips.
|
||||
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 3: common confusions.
|
||||
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
|
||||
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
|
||||
for (from, to) in rep {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 4: a missing space; both halves must be dictionary words.
|
||||
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
if let Some(cost) = split_cost(split, a, b) {
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
|
||||
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
|
||||
for row in col.d.key().split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [a, b] = pair else { continue };
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let replacement = if c == *a {
|
||||
Some(*b)
|
||||
} else if c == *b {
|
||||
Some(*a)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = replacement;
|
||||
}
|
||||
col.offer(&v.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
|
||||
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
|
||||
/// equal-cost candidates come out in that order.
|
||||
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
|
||||
let d = col.d;
|
||||
let n = chars.len();
|
||||
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
|
||||
let mut letters: Vec<char> = Vec::new();
|
||||
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
|
||||
if !letters.contains(&c) {
|
||||
letters.push(c);
|
||||
}
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.to_vec();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.to_vec();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.to_vec();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
let d = col.d;
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
|
||||
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
|
||||
let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT);
|
||||
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
|
||||
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
|
||||
}
|
||||
|
||||
/// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop)
|
||||
/// ranks at the lower one.
|
||||
#[test]
|
||||
fn a_cheaper_later_offer_wins() {
|
||||
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
|
||||
let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
|
||||
col.offer("cafe", COST_DROP_DIACRITIC);
|
||||
col.offer("cafe", COST_REP);
|
||||
col.offer("cafe", COST_SCAN);
|
||||
assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distance_weighs_map_slips_half() {
|
||||
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
|
||||
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
||||
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
||||
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
|
||||
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
|
||||
}
|
||||
|
||||
/// A KEEPCASE word is offered in its dictionary case when the word's own case would be flagged
|
||||
/// (`Kmm` → `km`, never `Km`), and a capitalised KEEPCASE word gets its dictionary form.
|
||||
#[test]
|
||||
fn keepcase_suggestions_are_not_recased_into_errors() {
|
||||
let d = AffixDict::from_hunspell("KEEPCASE K\n", "2\nkm/K\npies\n", &[]).unwrap();
|
||||
assert!(!d.check("Km") && !d.check("KM"));
|
||||
for (word, want) in [("Kmm", "km"), ("KMM", "km"), ("Km", "km")] {
|
||||
let got = suggest_in(&d, word, 4, &Tuning::DEFAULT);
|
||||
assert!(got.iter().all(|s| s.split(' ').all(|p| d.check(p))), "{word}: {got:?}");
|
||||
assert_eq!(got.first().map(String::as_str), Some(want), "{word}: {got:?}");
|
||||
}
|
||||
assert_eq!(suggest_in(&d, "Pies", 4, &Tuning::DEFAULT), Vec::<String>::new());
|
||||
assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies"));
|
||||
}
|
||||
|
||||
/// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY
|
||||
/// line, and a split rule decides which missing spaces are offered.
|
||||
#[test]
|
||||
fn tuning_adds_to_the_dictionary_tables() {
|
||||
let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap();
|
||||
let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT };
|
||||
assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into()));
|
||||
assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group");
|
||||
assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair");
|
||||
assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters");
|
||||
let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT };
|
||||
assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into()));
|
||||
assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half");
|
||||
assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into()));
|
||||
}
|
||||
}
|
||||
+10
-75
@@ -7,88 +7,35 @@ pub(crate) mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Czech Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
|
||||
|
||||
/// The Czech dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Czech");
|
||||
|
||||
/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Czech spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Czech dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals
|
||||
@@ -112,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Czech dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
@@ -1,168 +1,8 @@
|
||||
//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits
|
||||
//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian
|
||||
//! ([`crate::id`]) uses the same ranking with its own dictionary.
|
||||
//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed
|
||||
//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Longest word suggestions are looked for, in letters (Spanish shares the bound).
|
||||
pub(crate) const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 17;
|
||||
const COST_KEY: u32 = 13;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (a capitalized name).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (typing without diacritics); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::Tuning;
|
||||
|
||||
/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -171,237 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
let map = d.map();
|
||||
add_case_suggestions(&mut col, &input, word);
|
||||
add_map_suggestions(&mut col, &chars, map);
|
||||
add_replacement_suggestions(&mut col, &input, d.rep());
|
||||
add_split_suggestions(&mut col, &input);
|
||||
add_keyboard_suggestions(&mut col, &chars, d.key());
|
||||
add_typing_suggestions(&mut col, &chars, d.try_chars());
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (the first changed letter breaks ties).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn add_case_suggestions(col: &mut Collector, input: &str, word: &str) {
|
||||
if col.d.contains(input) && word != input {
|
||||
col.keep(input.to_string(), COST_CASE);
|
||||
} else {
|
||||
let capitalized = capitalize(input);
|
||||
if col.d.contains(&capitalized) && word != capitalized {
|
||||
col.keep(capitalized, COST_CASE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_map_suggestions(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
let alternatives: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alternatives, 0, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
variants.sort_by_key(|variant| variant.1);
|
||||
for (variant, cost) in variants {
|
||||
col.offer(&variant, cost);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_replacement_suggestions(col: &mut Collector, input: &str, replacements: &[(String, String)]) {
|
||||
for (from, to) in replacements {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (index, _) in input.match_indices(from) {
|
||||
if let (Some(prefix), Some(suffix)) = (input.get(..index), input.get(index + from.len()..)) {
|
||||
col.offer(&format!("{prefix}{to}{suffix}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_split_suggestions(col: &mut Collector, input: &str) {
|
||||
for (index, _) in input.char_indices().skip(1) {
|
||||
let (Some(first), Some(second)) = (input.get(..index), input.get(index..)) else { continue };
|
||||
let cost = if first.chars().count() >= 3 && second.chars().count() >= 3 { COST_SPLIT } else { COST_SPLIT_RARE };
|
||||
col.offer(&format!("{first} {second}"), cost);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_keyboard_suggestions(col: &mut Collector, chars: &[char], key_rows: &str) {
|
||||
for row in key_rows.split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [first, second] = pair else { continue };
|
||||
for (index, &character) in chars.iter().enumerate() {
|
||||
let replacement = if character == *first {
|
||||
Some(*second)
|
||||
} else if character == *second {
|
||||
Some(*first)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut candidate = chars.to_vec();
|
||||
if let Some(character) = candidate.get_mut(index) {
|
||||
*character = replacement;
|
||||
}
|
||||
col.offer(&candidate.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_typing_suggestions(col: &mut Collector, chars: &[char], try_chars: &str) {
|
||||
let mut letters: Vec<char> = try_chars.chars().filter(|character| character.is_lowercase()).take(128).collect();
|
||||
letters.sort_unstable();
|
||||
letters.dedup();
|
||||
let word = |candidate: &[char]| candidate.iter().collect::<String>();
|
||||
add_swap_suggestions(col, chars, &word);
|
||||
add_drop_suggestions(col, chars, &word);
|
||||
add_insert_suggestions(col, chars, &letters, &word);
|
||||
add_mistype_suggestions(col, chars, &letters, &word);
|
||||
}
|
||||
|
||||
fn add_swap_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len().saturating_sub(1) {
|
||||
if chars.get(index) != chars.get(index + 1) {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.swap(index, index + 1);
|
||||
col.offer(&word(&candidate), COST_SWAP);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_drop_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len() {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.remove(index);
|
||||
col.offer(&word(&candidate), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_insert_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..=chars.len() {
|
||||
for &letter in letters {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.insert(index, letter);
|
||||
col.offer(&word(&candidate), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_mistype_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len() {
|
||||
for &letter in letters {
|
||||
if chars.get(index) != Some(&letter) {
|
||||
let mut candidate = chars.to_vec();
|
||||
if let Some(character) = candidate.get_mut(index) {
|
||||
*character = letter;
|
||||
}
|
||||
col.offer(&word(&candidate), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
|
||||
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
|
||||
let suggestions = suggest_in(&d, "počítačz", 8);
|
||||
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
|
||||
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggestions_are_fast_enough() {
|
||||
crate::cs::test_dict::install();
|
||||
|
||||
+21
-76
@@ -16,16 +16,19 @@ mod suggest;
|
||||
pub(crate) mod test_dict;
|
||||
pub mod voseo;
|
||||
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::ProofLang;
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first
|
||||
/// use, then the installed one (or an empty one).
|
||||
static DICTS: [RwLock<Option<Arc<AffixDict>>>; 4] = [const { RwLock::new(None) }; 4];
|
||||
/// The dictionary of each variety, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static SPAIN_DICT: DictSlot<AffixDict> = DictSlot::new(SPAIN, "Spanish (Spain)");
|
||||
pub(crate) static MEXICO_DICT: DictSlot<AffixDict> = DictSlot::new(MEXICO, "Spanish (Mexico)");
|
||||
pub(crate) static LATIN_AMERICA_DICT: DictSlot<AffixDict> = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)");
|
||||
pub(crate) static URUGUAY_DICT: DictSlot<AffixDict> = DictSlot::new(URUGUAY, "Spanish (Uruguay)");
|
||||
|
||||
/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the
|
||||
/// generic one RLA-ES publishes).
|
||||
@@ -40,84 +43,38 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e
|
||||
/// or generic one.
|
||||
const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"];
|
||||
|
||||
/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's
|
||||
/// file names; Spain for any other language.
|
||||
fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) {
|
||||
/// `lang`'s variety's dictionary; Spain's for any other language.
|
||||
fn slot(lang: ProofLang) -> &'static DictSlot<AffixDict> {
|
||||
match lang {
|
||||
ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO),
|
||||
ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA),
|
||||
ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY),
|
||||
_ => (0, "Spanish (Spain)", SPAIN),
|
||||
ProofLang::EsMx => &MEXICO_DICT,
|
||||
ProofLang::Es419 => &LATIN_AMERICA_DICT,
|
||||
ProofLang::EsUy => &URUGUAY_DICT,
|
||||
_ => &SPAIN_DICT,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`,
|
||||
/// `es_UY`): the first of its file names, used for the status-bar hint and load errors.
|
||||
pub fn dictionary_name(lang: ProofLang) -> &'static str {
|
||||
variety(lang).2.first().copied().unwrap_or("es_ES")
|
||||
slot(lang).name()
|
||||
}
|
||||
|
||||
/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given
|
||||
/// with [`install_dictionary`], or an empty one when none is available (spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
pub fn dictionary(lang: ProofLang) -> Arc<AffixDict> {
|
||||
let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) };
|
||||
if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed(lang));
|
||||
let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed(lang: ProofLang) -> AffixDict {
|
||||
let (_, name, files) = variety(lang);
|
||||
let key = dictionary_name(lang);
|
||||
let Some((aff, dic)) = crate::hunspell::find(files) else {
|
||||
crate::hunspell::clear_load_error(key);
|
||||
log::info!("{name} spelling: no Hunspell {key} dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("{name} spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
slot(lang).get()
|
||||
}
|
||||
|
||||
/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text,
|
||||
/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
if let Some(slot) = DICTS.get(variety(lang).0) {
|
||||
*slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget every variety's loaded dictionary, so the next check looks for an installed one again
|
||||
/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
for slot in &DICTS {
|
||||
*slot.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder
|
||||
/// or the system locations now (File › Options › Proofing shows it); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path(lang: ProofLang) -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic)
|
||||
slot(lang).install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a dictionary available for `lang`'s variety (spelling is checked)?
|
||||
pub fn has_dictionary(lang: ProofLang) -> bool {
|
||||
!dictionary(lang).is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
slot(lang).has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in
|
||||
@@ -143,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Spanish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang));
|
||||
}
|
||||
@@ -200,7 +145,7 @@ mod tests {
|
||||
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
|
||||
for lang in VARIETIES {
|
||||
let found = crate::hunspell::find_in(&dir, variety(lang).2);
|
||||
let found = crate::hunspell::find_in(&dir, slot(lang).names());
|
||||
assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}");
|
||||
if let Some((aff, dic)) = found {
|
||||
assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang));
|
||||
@@ -215,7 +160,7 @@ mod tests {
|
||||
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
|
||||
}
|
||||
for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] {
|
||||
assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
|
||||
assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
|
||||
assert_eq!(dictionary_name(lang), name);
|
||||
}
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses
|
||||
//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to
|
||||
//! Spanish suggestions. The ranking is the shared dictionary-driven one
|
||||
//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to
|
||||
//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces,
|
||||
//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary.
|
||||
//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use crate::ProofLang;
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::cs::suggest::MAX_LETTERS;
|
||||
use crate::affix_suggest::{MAX_LETTERS, Tuning};
|
||||
|
||||
/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's
|
||||
/// case.
|
||||
@@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec
|
||||
_ => v,
|
||||
});
|
||||
}
|
||||
for s in crate::cs::suggest::suggest_in(d, word, max) {
|
||||
for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) {
|
||||
if !out.contains(&s) {
|
||||
out.push(s);
|
||||
}
|
||||
|
||||
+10
-60
@@ -9,73 +9,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Croatian Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"];
|
||||
|
||||
/// The Croatian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Croatian");
|
||||
|
||||
/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Croatian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Croatian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals
|
||||
@@ -99,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Croatian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
+25
-327
@@ -1,12 +1,11 @@
|
||||
//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance,
|
||||
//! missing spaces and a bounded search. Proper Croatian corrections depend on
|
||||
//! the actual installed dictionary entries.
|
||||
//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of
|
||||
//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
|
||||
/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
"a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će",
|
||||
"bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad",
|
||||
@@ -14,159 +13,13 @@ const LEADING_WORDS: &[&str] = &[
|
||||
const TRAILING_WORDS: &[&str] =
|
||||
&["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"];
|
||||
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table
|
||||
/// (`seje` → `seja` before `sejé`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
/// What Croatian adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aeiourntsjkvlmdpbgczčćšžđfh",
|
||||
map: &["cčć", "dđ", "sš", "zž"],
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -175,179 +28,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']];
|
||||
let map = if d.map().is_empty() { &fallback_map } else { d.map() };
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (`saúde` before `saudê`: both add an accent, but at the first difference).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn nosuggest_stems_and_forms_are_never_offered() {
|
||||
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
|
||||
}
|
||||
/// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća`
|
||||
/// first, before a typo that costs more (`kuha`).
|
||||
#[test]
|
||||
fn diacritics_without_map_lines() {
|
||||
let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap();
|
||||
assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]);
|
||||
}
|
||||
#[test]
|
||||
fn croatian_suggestions() {
|
||||
crate::hr::test_dict::install();
|
||||
for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] {
|
||||
|
||||
@@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
|
||||
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
|
||||
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
|
||||
/// and Windows-1251 the ones older Russian dictionaries use).
|
||||
///
|
||||
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
|
||||
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
|
||||
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
|
||||
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
|
||||
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
|
||||
/// its number (the Turkish dictionary's `FLAG num`).
|
||||
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
|
||||
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
|
||||
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
|
||||
let (set, flag) = directives(aff);
|
||||
let set = set.unwrap_or_else(|| "UTF-8".into());
|
||||
match set.as_str() {
|
||||
"UTF-8" | "UTF8" => {
|
||||
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
|
||||
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
|
||||
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
|
||||
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
|
||||
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
|
||||
}
|
||||
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
|
||||
}
|
||||
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
|
||||
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
|
||||
@@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
|
||||
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
|
||||
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
|
||||
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
|
||||
let (mut set, mut flag) = (None, None);
|
||||
for l in aff.split(|&b| b == b'\n') {
|
||||
if set.is_some() && flag.is_some() {
|
||||
break;
|
||||
}
|
||||
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
|
||||
let slot = match l.first() {
|
||||
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
|
||||
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
|
||||
_ => continue,
|
||||
};
|
||||
let arg = l
|
||||
.strip_prefix(slot.1)
|
||||
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
|
||||
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
|
||||
if let Some(a) = arg {
|
||||
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
|
||||
}
|
||||
}
|
||||
(set, flag)
|
||||
}
|
||||
|
||||
/// Directives whose one argument is a flag.
|
||||
const FLAG_DIRECTIVES: &[&str] = &[
|
||||
"CIRCUMFIX",
|
||||
"COMPOUNDBEGIN",
|
||||
"COMPOUNDEND",
|
||||
"COMPOUNDFLAG",
|
||||
"COMPOUNDFORBIDFLAG",
|
||||
"COMPOUNDLAST",
|
||||
"COMPOUNDFIRST",
|
||||
"COMPOUNDMIDDLE",
|
||||
"COMPOUNDPERMITFLAG",
|
||||
"COMPOUNDROOT",
|
||||
"FORBIDDENWORD",
|
||||
"FORCEUCASE",
|
||||
"GENERATE",
|
||||
"HU_KOTOHANGZO",
|
||||
"KEEPCASE",
|
||||
"LEMMA_PRESENT",
|
||||
"NEEDAFFIX",
|
||||
"NOSUGGEST",
|
||||
"ONLYINCOMPOUND",
|
||||
"ONLYROOT",
|
||||
"PSEUDOROOT",
|
||||
"SUBSTANDARD",
|
||||
"WARN",
|
||||
];
|
||||
|
||||
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
|
||||
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
|
||||
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
|
||||
|
||||
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
|
||||
/// names the line in an error.
|
||||
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
|
||||
let mut slash = None;
|
||||
let mut i = 0;
|
||||
while let Some(&b) = field.get(i) {
|
||||
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if b == b'/' {
|
||||
slash = Some(i);
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
let (text, flags) = match slash {
|
||||
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
|
||||
None => (field, &[][..]),
|
||||
};
|
||||
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
|
||||
out.push_str(&decode_latin1(flags));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
|
||||
/// its parts is the original).
|
||||
fn fields(line: &[u8]) -> Vec<&[u8]> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
|
||||
for (i, b) in line.iter().enumerate() {
|
||||
let sep = matches!(b, b' ' | b'\t');
|
||||
if sep != in_sep {
|
||||
out.push(line.get(start..i).unwrap_or_default());
|
||||
start = i;
|
||||
in_sep = sep;
|
||||
}
|
||||
}
|
||||
out.push(line.get(start..).unwrap_or_default());
|
||||
out
|
||||
}
|
||||
|
||||
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
|
||||
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
|
||||
let mut out = String::with_capacity(aff.len());
|
||||
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
|
||||
if n > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
if let Ok(s) = std::str::from_utf8(line)
|
||||
&& s.is_ascii()
|
||||
{
|
||||
out.push_str(s);
|
||||
continue;
|
||||
}
|
||||
let parts = fields(line);
|
||||
// Field index among the non-separator parts.
|
||||
let words: Vec<usize> =
|
||||
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
|
||||
let dir = words
|
||||
.first()
|
||||
.and_then(|&i| parts.get(i))
|
||||
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
|
||||
.unwrap_or_default();
|
||||
let what = || format!(".aff line {}", n + 1);
|
||||
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
|
||||
out.push_str(&String::from_utf8_lossy(line));
|
||||
continue;
|
||||
}
|
||||
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
|
||||
// a flag; from there on the line is kept as text. It can only start after the directive's
|
||||
// own fields, since a field such as an affix condition may itself begin with `#`.
|
||||
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
|
||||
let first_free = comment_field(&dir, field_text);
|
||||
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
|
||||
// The field index of the next non-separator part.
|
||||
let mut next_field = 0;
|
||||
for (i, p) in parts.iter().enumerate() {
|
||||
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
|
||||
next_field += 1;
|
||||
next_field - 1
|
||||
});
|
||||
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
|
||||
let decoded = match (dir.as_str(), field) {
|
||||
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
|
||||
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
|
||||
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
|
||||
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
|
||||
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
|
||||
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
|
||||
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
|
||||
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
|
||||
};
|
||||
out.push_str(&decoded);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
|
||||
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
|
||||
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
|
||||
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
|
||||
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
|
||||
match dir {
|
||||
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
|
||||
"PFX" | "SFX" => 5,
|
||||
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
|
||||
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
|
||||
{
|
||||
2
|
||||
}
|
||||
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
|
||||
"CHECKCOMPOUNDPATTERN" => 4,
|
||||
_ => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
|
||||
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
|
||||
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
|
||||
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
|
||||
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
|
||||
let blank = |b: &u8| matches!(b, b' ' | b'\t');
|
||||
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
|
||||
let tab = line.iter().position(|&b| b == b'\t');
|
||||
let end = match (morph, tab) {
|
||||
(Some(m), Some(t)) => m.min(t),
|
||||
(m, t) => match m.or(t) {
|
||||
Some(e) => e,
|
||||
None => line.len(),
|
||||
},
|
||||
};
|
||||
// Blanks before the morphological field or the line end aren't part of the entry.
|
||||
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
|
||||
}
|
||||
|
||||
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
|
||||
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
|
||||
let mut out = String::with_capacity(dic.len());
|
||||
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
|
||||
if n > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
if let Ok(s) = std::str::from_utf8(line)
|
||||
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
|
||||
{
|
||||
out.push_str(s);
|
||||
continue;
|
||||
}
|
||||
let what = || format!(".dic line {}", n + 1);
|
||||
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
|
||||
// morphological fields after it are UTF-8 text.
|
||||
let end = dic_entry_end(line);
|
||||
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
|
||||
let rest = line.get(end..).unwrap_or_default();
|
||||
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -280,6 +509,78 @@ mod tests {
|
||||
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_dictionaries_keep_single_byte_flags() {
|
||||
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
|
||||
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
|
||||
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
|
||||
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
|
||||
let (a, d) = decode(aff, dic).unwrap();
|
||||
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
|
||||
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
|
||||
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
|
||||
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
|
||||
assert!(!dict.check("házban"));
|
||||
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
|
||||
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||||
assert_eq!(d, "kert/\u{e9}\n");
|
||||
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
|
||||
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_flag_fields_keep_their_first_byte() {
|
||||
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
|
||||
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
|
||||
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
|
||||
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
|
||||
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
|
||||
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
|
||||
assert!(d.contains("kert/\u{c3}\u{a9}"));
|
||||
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
|
||||
assert!(dict.check("kert") && dict.check("kertok"));
|
||||
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_and_long_flags_stay_text() {
|
||||
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
|
||||
// number with a non-ASCII rule or comment keeps every digit.
|
||||
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
|
||||
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
|
||||
assert_eq!(a, aff);
|
||||
assert_eq!(d, "1\nkitap/1234\n");
|
||||
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
|
||||
assert!(tr.contains("kitapları") && tr.contains("kitap"));
|
||||
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
|
||||
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
|
||||
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
|
||||
// `FLAG DEFAULT` is the single-byte default.
|
||||
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
|
||||
assert_eq!(d, "1\nkert/\u{e9}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flag_utf8_counts_anywhere_in_the_aff() {
|
||||
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
|
||||
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
|
||||
for _ in 0..400 {
|
||||
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
|
||||
}
|
||||
aff.extend_from_slice(b"FLAG UTF-8\r\n");
|
||||
assert!(aff.len() > 8192);
|
||||
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
|
||||
assert_eq!(d, "kert/\u{e9}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
|
||||
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||||
assert_eq!(d, "kert/\u{e9}\n");
|
||||
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||||
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_russian_single_byte_encodings() {
|
||||
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
|
||||
@@ -291,6 +592,41 @@ mod tests {
|
||||
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_start_after_the_directive_fields() {
|
||||
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
|
||||
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
|
||||
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
|
||||
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
|
||||
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
|
||||
assert_eq!(comment_field("REP", f("REP 12")), 2);
|
||||
assert_eq!(comment_field("REP", f("REP # x")), 3);
|
||||
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
|
||||
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
|
||||
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
|
||||
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
|
||||
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dic_entries_end_where_hunspell_ends_them() {
|
||||
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
|
||||
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
|
||||
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
|
||||
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
|
||||
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
|
||||
assert_eq!(end("ad hoc/B "), "ad hoc/B");
|
||||
assert_eq!(end("word"), "word");
|
||||
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
|
||||
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
|
||||
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
|
||||
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
|
||||
// Without a tab or a morphological field the whole line is the entry: its byte flags
|
||||
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
|
||||
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
|
||||
assert_eq!(d, "1\nad hoc/\u{e9}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libreoffice_extension_dirs_follow_the_language() {
|
||||
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
|
||||
|
||||
+12
-61
@@ -9,78 +9,41 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
|
||||
/// Java locales).
|
||||
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
|
||||
|
||||
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
|
||||
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
|
||||
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
|
||||
/// The Indonesian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Indonesian");
|
||||
|
||||
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Indonesian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it); `None` when there is none. A
|
||||
/// dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is an Indonesian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
|
||||
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
|
||||
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
|
||||
|
||||
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
|
||||
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Indonesian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return known(d, w)
|
||||
|| w.split('-').all(|p| {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses
|
||||
//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem
|
||||
//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is
|
||||
//! hard-coded here.
|
||||
//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing
|
||||
//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID`
|
||||
//! dictionary, so nothing Indonesian is hard-coded here.
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
@@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
crate::cs::suggest::suggest_in(d, word, max)
|
||||
crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+10
-59
@@ -18,72 +18,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Italian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
|
||||
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
|
||||
|
||||
/// The Italian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Italian");
|
||||
|
||||
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Italian spelling: no Hunspell it_IT dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Italian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is an Italian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
|
||||
@@ -125,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Italian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
let lower = w.to_lowercase();
|
||||
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
|
||||
return true;
|
||||
|
||||
+16
-336
@@ -1,25 +1,9 @@
|
||||
//! Italian spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`roma` → `Roma`);
|
||||
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`,
|
||||
//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the
|
||||
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`);
|
||||
//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which an accent slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
|
||||
//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and
|
||||
//! letters to try for a dictionary without a TRY line.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
@@ -35,158 +19,12 @@ const TRAILING_WORDS: &[&str] = &[
|
||||
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
|
||||
"è",
|
||||
];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance.
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
/// What Italian adds to its dictionary's tables.
|
||||
const TUNING: Tuning =
|
||||
Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `à` → `a`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -195,182 +33,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent.
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nosuggest_stems_and_forms_are_never_offered() {
|
||||
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_misspellings_get_the_right_suggestion() {
|
||||
crate::it::test_dict::install();
|
||||
|
||||
+197
-135
@@ -50,6 +50,7 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod affix;
|
||||
mod affix_suggest;
|
||||
pub mod bg;
|
||||
pub mod cs;
|
||||
pub mod dict;
|
||||
@@ -65,6 +66,8 @@ pub mod patterns;
|
||||
pub mod pl;
|
||||
pub mod pt;
|
||||
pub mod ru;
|
||||
mod slot;
|
||||
mod spelling;
|
||||
pub mod tr;
|
||||
pub mod vi;
|
||||
|
||||
@@ -157,57 +160,150 @@ pub fn ignore_word(w: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// A language proofed against an installed Hunspell dictionary: all the functions below need to
|
||||
/// know about it, so a language joins with one row in [`LANGUAGES`].
|
||||
struct Language {
|
||||
lang: ProofLang,
|
||||
slot: &'static dyn slot::Slot,
|
||||
/// The spelling mark's message, in the language.
|
||||
message: &'static str,
|
||||
is_correct: fn(&str, ProofLang, &Options) -> bool,
|
||||
suggest: fn(&str, usize, ProofLang) -> Vec<String>,
|
||||
/// Grammar issues; the flag says whether the first word must have a capital (not in a list
|
||||
/// item).
|
||||
grammar: fn(&str, ProofLang, bool) -> Vec<Issue>,
|
||||
}
|
||||
|
||||
/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing
|
||||
/// lists them. English has a built-in word list and is handled apart.
|
||||
static LANGUAGES: [Language; 12] = [
|
||||
Language {
|
||||
lang: ProofLang::Pl,
|
||||
slot: &pl::DICT,
|
||||
message: "Możliwy błąd pisowni",
|
||||
is_correct: |w, _, o| pl::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| pl::suggest(w, n),
|
||||
grammar: |t, _, c| pl::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Pt,
|
||||
slot: &pt::DICT,
|
||||
message: "Possível erro de ortografia",
|
||||
is_correct: |w, _, o| pt::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| pt::suggest(w, n),
|
||||
grammar: |t, _, c| pt::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::It,
|
||||
slot: &it::DICT,
|
||||
message: "Possibile errore di ortografia",
|
||||
is_correct: |w, _, o| it::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| it::suggest(w, n),
|
||||
grammar: |t, _, c| it::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Hr,
|
||||
slot: &hr::DICT,
|
||||
message: "Moguća pravopisna pogreška",
|
||||
is_correct: |w, _, o| hr::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| hr::suggest(w, n),
|
||||
grammar: |t, _, c| hr::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Tr,
|
||||
slot: &tr::DICT,
|
||||
message: "Olası yazım hatası",
|
||||
is_correct: |w, _, o| tr::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| tr::suggest(w, n),
|
||||
grammar: |t, _, c| tr::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Ru,
|
||||
slot: &ru::DICT,
|
||||
message: "Возможна орфографическая ошибка",
|
||||
is_correct: |w, _, o| ru::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| ru::suggest(w, n),
|
||||
grammar: |t, _, c| ru::grammar::check_with(t, c),
|
||||
},
|
||||
// Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no
|
||||
// exception.
|
||||
Language {
|
||||
lang: ProofLang::Cs,
|
||||
slot: &cs::DICT,
|
||||
message: "Možná pravopisná chyba",
|
||||
is_correct: |w, _, o| cs::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| cs::suggest(w, n),
|
||||
grammar: |t, _, _| cs::grammar::check(t),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Id,
|
||||
slot: &id::DICT,
|
||||
message: "Kemungkinan salah eja",
|
||||
is_correct: |w, _, o| id::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| id::suggest(w, n),
|
||||
grammar: |t, _, _| id::grammar::check(t),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Es,
|
||||
slot: &es::SPAIN_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::EsMx,
|
||||
slot: &es::MEXICO_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Es419,
|
||||
slot: &es::LATIN_AMERICA_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::EsUy,
|
||||
slot: &es::URUGUAY_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
];
|
||||
|
||||
/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof.
|
||||
fn language(lang: ProofLang) -> Option<&'static Language> {
|
||||
LANGUAGES.iter().find(|l| l.lang == lang)
|
||||
}
|
||||
|
||||
/// Search `dir` for Hunspell dictionaries before the system locations (File › Options ›
|
||||
/// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from
|
||||
/// the new search path, and text is checked again.
|
||||
pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
|
||||
hunspell::set_dictionary_dir(dir);
|
||||
pl::reload_dictionary();
|
||||
pt::reload_dictionary();
|
||||
it::reload_dictionary();
|
||||
hr::reload_dictionary();
|
||||
tr::reload_dictionary();
|
||||
ru::reload_dictionary();
|
||||
cs::reload_dictionary();
|
||||
id::reload_dictionary();
|
||||
es::reload_dictionary();
|
||||
for l in &LANGUAGES {
|
||||
l.slot.reload();
|
||||
}
|
||||
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it
|
||||
/// now (`None`: not installed, so the language has no spelling marks).
|
||||
pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)> {
|
||||
vec![
|
||||
(ProofLang::Pl, pl::installed_path()),
|
||||
(ProofLang::Pt, pt::installed_path()),
|
||||
(ProofLang::It, it::installed_path()),
|
||||
(ProofLang::Hr, hr::installed_path()),
|
||||
(ProofLang::Tr, tr::installed_path()),
|
||||
(ProofLang::Ru, ru::installed_path()),
|
||||
(ProofLang::Cs, cs::installed_path()),
|
||||
(ProofLang::Id, id::installed_path()),
|
||||
(ProofLang::Es, es::installed_path(ProofLang::Es)),
|
||||
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
|
||||
(ProofLang::Es419, es::installed_path(ProofLang::Es419)),
|
||||
(ProofLang::EsUy, es::installed_path(ProofLang::EsUy)),
|
||||
]
|
||||
LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect()
|
||||
}
|
||||
|
||||
/// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse
|
||||
/// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has
|
||||
/// checked yet reports `None` (#2830).
|
||||
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
|
||||
let key = match lang {
|
||||
ProofLang::Pl => "pl_PL",
|
||||
ProofLang::Pt => "pt_BR",
|
||||
ProofLang::It => "it_IT",
|
||||
ProofLang::Hr => "hr_HR",
|
||||
ProofLang::Tr => "tr_TR",
|
||||
ProofLang::Id => "id_ID",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang),
|
||||
_ => return None,
|
||||
};
|
||||
hunspell::load_error(key).map(|(_, reason)| reason)
|
||||
hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason)
|
||||
}
|
||||
|
||||
/// A number that changes whenever a word is added to or removed from the user dictionary, or the
|
||||
@@ -438,36 +534,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p
|
||||
|
||||
/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English).
|
||||
fn dictionary_loaded(lang: ProofLang) -> Option<bool> {
|
||||
Some(match lang {
|
||||
ProofLang::Pl => pl::has_dictionary(),
|
||||
ProofLang::Pt => pt::has_dictionary(),
|
||||
ProofLang::It => it::has_dictionary(),
|
||||
ProofLang::Hr => hr::has_dictionary(),
|
||||
ProofLang::Tr => tr::has_dictionary(),
|
||||
ProofLang::Ru => ru::has_dictionary(),
|
||||
ProofLang::Cs => cs::has_dictionary(),
|
||||
ProofLang::Id => id::has_dictionary(),
|
||||
l if l.is_spanish() => es::has_dictionary(l),
|
||||
_ => return None,
|
||||
})
|
||||
language(lang).map(|l| l.slot.has_dictionary())
|
||||
}
|
||||
|
||||
/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for;
|
||||
/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`).
|
||||
fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> {
|
||||
match lang {
|
||||
_ if installed => None,
|
||||
ProofLang::Pl => Some("pl_PL"),
|
||||
ProofLang::Pt => Some("pt_BR"),
|
||||
ProofLang::It => Some("it_IT"),
|
||||
ProofLang::Hr => Some("hr_HR"),
|
||||
ProofLang::Tr => Some("tr_TR"),
|
||||
ProofLang::Ru => Some("ru_RU"),
|
||||
ProofLang::Cs => Some("cs_CZ"),
|
||||
ProofLang::Id => Some("id_ID"),
|
||||
l if l.is_spanish() => Some(es::dictionary_name(l)),
|
||||
_ => None,
|
||||
}
|
||||
language(lang).filter(|_| !installed).map(|l| l.slot.name())
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary
|
||||
@@ -739,18 +812,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
|
||||
if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) {
|
||||
return None;
|
||||
}
|
||||
let message = match lang {
|
||||
ProofLang::Pl => "Możliwy błąd pisowni",
|
||||
ProofLang::Hr => "Moguća pravopisna pogreška",
|
||||
ProofLang::Pt => "Possível erro de ortografia",
|
||||
ProofLang::It => "Possibile errore di ortografia",
|
||||
ProofLang::Tr => "Olası yazım hatası",
|
||||
ProofLang::Ru => "Возможна орфографическая ошибка",
|
||||
ProofLang::Cs => "Možná pravopisná chyba",
|
||||
ProofLang::Id => "Kemungkinan salah eja",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico",
|
||||
_ => "Possible spelling mistake",
|
||||
};
|
||||
let message = language(lang).map_or("Possible spelling mistake", |l| l.message);
|
||||
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
|
||||
}
|
||||
|
||||
@@ -774,40 +836,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> {
|
||||
/// checked as its composed form ([`composed`]).
|
||||
pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
let word = &*composed(word);
|
||||
match lang {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts),
|
||||
ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts),
|
||||
ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts),
|
||||
ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts),
|
||||
ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts),
|
||||
ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word),
|
||||
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => {
|
||||
accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts)
|
||||
}
|
||||
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts),
|
||||
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
|
||||
if lang.is_english() {
|
||||
return is_correct_english(word, lang, opts);
|
||||
}
|
||||
// A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks.
|
||||
language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts))
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed
|
||||
/// ([`composed`]) like the dictionaries' words.
|
||||
pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
let word = &*composed(word);
|
||||
match lang {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang),
|
||||
ProofLang::Pl => pl::suggest(word, max),
|
||||
ProofLang::Hr => hr::suggest(word, max),
|
||||
ProofLang::Pt => pt::suggest(word, max),
|
||||
ProofLang::It => it::suggest(word, max),
|
||||
ProofLang::Tr => tr::suggest(word, max),
|
||||
ProofLang::Ru => ru::suggest(word, max),
|
||||
ProofLang::Cs => cs::suggest(word, max),
|
||||
ProofLang::Id => id::suggest(word, max),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
if lang.is_english() {
|
||||
return suggest_english(word, max, lang);
|
||||
}
|
||||
language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang))
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
|
||||
@@ -824,18 +867,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
|
||||
match lang {
|
||||
ProofLang::En => grammar_en(text, capital_first),
|
||||
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
|
||||
ProofLang::Pl => pl::grammar::check_with(text, capital_first),
|
||||
ProofLang::Hr => hr::grammar::check_with(text, capital_first),
|
||||
ProofLang::Pt => pt::grammar::check_with(text, capital_first),
|
||||
ProofLang::It => it::grammar::check_with(text, capital_first),
|
||||
ProofLang::Tr => tr::grammar::check_with(text, capital_first),
|
||||
ProofLang::Ru => ru::grammar::check_with(text, capital_first),
|
||||
// Czech has no sentence-capital rule, so a list item needs no exception.
|
||||
ProofLang::Cs => cs::grammar::check(text),
|
||||
ProofLang::Id => id::grammar::check(text),
|
||||
// Spanish has no sentence-capital rule either.
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
_ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1060,7 +1092,9 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test.
|
||||
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land
|
||||
/// mid-test, and by those that save the whole user dictionary and load it back, which would
|
||||
/// bring back a word another test just removed.
|
||||
static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// #1063: the dictionaries folder is searched before the system locations; the dictionaries
|
||||
@@ -1077,6 +1111,7 @@ mod tests {
|
||||
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
|
||||
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
|
||||
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
|
||||
("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC),
|
||||
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
|
||||
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
|
||||
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
|
||||
@@ -1096,41 +1131,67 @@ mod tests {
|
||||
let before = dictionary_generation();
|
||||
set_dictionaries_folder(Some(dir.clone()));
|
||||
assert!(dictionary_generation() > before, "cached checks are dropped");
|
||||
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
|
||||
// Every language is listed, Russian and Czech too (#3226: they were once missing, so
|
||||
// neither showed up in File › Options › Proofing). A language reports the folder's file,
|
||||
// or none while a dictionary given with install_dictionary() is in use (#3252 review);
|
||||
// the languages' own tests install theirs concurrently, so an exact path would race them.
|
||||
// The folder is searched first whatever is in use.
|
||||
let found = installed_dictionaries();
|
||||
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
|
||||
let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"];
|
||||
assert_eq!(
|
||||
found[7..],
|
||||
found.iter().map(|(lang, _)| *lang).collect::<Vec<_>>(),
|
||||
[
|
||||
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
|
||||
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
|
||||
(ProofLang::EsMx, Some(dir.join("es_MX.dic"))),
|
||||
(ProofLang::Es419, Some(dir.join("es_419.dic"))),
|
||||
(ProofLang::EsUy, Some(dir.join("es_UY.dic"))),
|
||||
ProofLang::Pl,
|
||||
ProofLang::Pt,
|
||||
ProofLang::It,
|
||||
ProofLang::Hr,
|
||||
ProofLang::Tr,
|
||||
ProofLang::Ru,
|
||||
ProofLang::Cs,
|
||||
ProofLang::Id,
|
||||
ProofLang::Es,
|
||||
ProofLang::EsMx,
|
||||
ProofLang::Es419,
|
||||
ProofLang::EsUy
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
found[..4],
|
||||
[
|
||||
(ProofLang::Pl, Some(dir.join("pl_PL.dic"))),
|
||||
(ProofLang::Pt, Some(dir.join("pt_BR.dic"))),
|
||||
(ProofLang::It, Some(dir.join("it_IT.dic"))),
|
||||
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
|
||||
]
|
||||
);
|
||||
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
|
||||
// installed_path() in either module), so neither ever showed up in File › Options ›
|
||||
// Proofing, installed or not. Their languages, not their exact path, is what belongs
|
||||
// here: unlike the four above, installed_path() for Russian and Czech also reflects
|
||||
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
|
||||
// test modules do, concurrently, in tests of their own — asserting an exact path here
|
||||
// would race them for no reason this test needs to care about.
|
||||
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
|
||||
for ((lang, path), name) in found.iter().zip(names) {
|
||||
let file = dir.join(format!("{name}.dic"));
|
||||
assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}");
|
||||
assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}");
|
||||
}
|
||||
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
|
||||
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
|
||||
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
|
||||
}
|
||||
|
||||
/// A dictionary that fails to load says why in File › Options › Proofing in every language
|
||||
/// that reads one; Russian and Czech were missing from the list that named their dictionaries
|
||||
/// there.
|
||||
#[test]
|
||||
fn every_dictionary_language_reports_its_load_error() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for (lang, _) in installed_dictionaries() {
|
||||
let name = dictionary_needed(lang, false).unwrap();
|
||||
let reason = format!("zq unreadable {name}");
|
||||
let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone()));
|
||||
assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}");
|
||||
hunspell::clear_load_error(name);
|
||||
}
|
||||
}
|
||||
|
||||
/// Every proofed language other than English has a row in the language table, and only those.
|
||||
#[test]
|
||||
fn the_language_table_covers_the_proofed_languages() {
|
||||
for l in ProofLang::PROOFED {
|
||||
assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}");
|
||||
}
|
||||
for (i, row) in LANGUAGES.iter().enumerate() {
|
||||
assert!(row.lang.is_proofed(), "{:?}", row.lang);
|
||||
assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang);
|
||||
}
|
||||
}
|
||||
|
||||
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
|
||||
/// with the rule; a plain paragraph's does, and later sentences in the item are still checked.
|
||||
#[test]
|
||||
@@ -1593,6 +1654,7 @@ mod tests {
|
||||
/// #40: the user dictionary survives a save and a fresh load.
|
||||
#[test]
|
||||
fn user_dictionary_persists_through_a_file() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
// The folder is created on the first save.
|
||||
|
||||
+14
-63
@@ -17,78 +17,41 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Polish dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Polish Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"];
|
||||
|
||||
/// The Polish dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Polish");
|
||||
|
||||
/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Polish spelling: no Hunspell pl_PL dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Polish spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
|
||||
/// for example one the user picked. On error the current dictionary stays.
|
||||
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Polish dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced
|
||||
/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`).
|
||||
const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"];
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so
|
||||
/// is every word when no Polish dictionary is installed.
|
||||
@@ -110,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Polish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
let w = w.replace('’', "'");
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
|
||||
+19
-342
@@ -1,26 +1,9 @@
|
||||
//! Polish spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`warszawa` → `Warszawa`);
|
||||
//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy`
|
||||
//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most
|
||||
//! common slip, so adding them ranks first, while taking away a diacritic the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` →
|
||||
//! `huśtawka`, `pszyjaciel` → `przyjaciel`);
|
||||
//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham
|
||||
//! cię`), cheaper when one half is a short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which a diacritic slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
|
||||
//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`),
|
||||
//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede
|
||||
/// wszystkim`).
|
||||
@@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[
|
||||
/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`).
|
||||
const TRAILING_WORDS: &[&str] =
|
||||
&["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
/// What Polish adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx",
|
||||
mistype_cost,
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moze` → `może`, then
|
||||
// `Mozę`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the
|
||||
/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the
|
||||
/// writer did type is likelier right, so taking one away costs more than a known confusion from
|
||||
/// the REP table (`żeka` → `rzeka` before `zeka`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it
|
||||
/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo.
|
||||
fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 {
|
||||
let last = i + 1 == chars.len();
|
||||
if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case.
|
||||
@@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Diacritics and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half (`że ka`).
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
// Final devoicing can make a voiced consonant sound voiceless.
|
||||
let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g'));
|
||||
let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE };
|
||||
col.offer(&s(&v), cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
found.sort_by_key(|(_, k)| *k);
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
|
||||
+10
-59
@@ -16,72 +16,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Portuguese Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"];
|
||||
|
||||
/// The Portuguese dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Portuguese");
|
||||
|
||||
/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Portuguese spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Portuguese dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals
|
||||
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Portuguese dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
+13
-343
@@ -1,26 +1,9 @@
|
||||
//! Portuguese spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`brasília` → `Brasília`);
|
||||
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`,
|
||||
//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the
|
||||
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` →
|
||||
//! `exceção`, `previlégio` → `privilégio`);
|
||||
//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which an accent slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and
|
||||
//! letters to try for a dictionary without a TRY line.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
@@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[
|
||||
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
|
||||
"eu", "tu", "que", "de", "em", "é",
|
||||
];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table
|
||||
/// (`seje` → `seja` before `sejé`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
/// What Portuguese adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw",
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (`saúde` before `saudê`: both add an accent, but at the first difference).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
|
||||
+51
-66
@@ -28,83 +28,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Russian Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
|
||||
|
||||
/// The Russian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Russian");
|
||||
|
||||
/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Russian spelling: no Hunspell ru_RU dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Russian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
|
||||
/// for example one the user picked. On error the current dictionary stays.
|
||||
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Russian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)?
|
||||
@@ -130,18 +82,30 @@ pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
|
||||
/// short words in capitals are checked against the dictionary too, and with Ignore words that
|
||||
/// contain numbers off, so are words with digits or superscripts.
|
||||
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
|
||||
check(&dictionary(), word, opts)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
|
||||
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
check(d, word, &crate::Options::default())
|
||||
}
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Russian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
|
||||
let w = w.trim_matches('-');
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) {
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
|
||||
return true;
|
||||
}
|
||||
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
|
||||
@@ -161,7 +125,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
return true;
|
||||
}
|
||||
if w.contains('-') {
|
||||
return hyphenated(d, w);
|
||||
return hyphenated(d, w, opts);
|
||||
}
|
||||
solid_compound(d, w)
|
||||
}
|
||||
@@ -340,7 +304,7 @@ fn known(d: &AffixDict, w: &str) -> bool {
|
||||
|
||||
/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles,
|
||||
/// prefixes and linking vowels Russian joins with a hyphen.
|
||||
fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
|
||||
let parts: Vec<&str> = w.split('-').collect();
|
||||
if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 {
|
||||
return false;
|
||||
@@ -353,7 +317,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
&& lp.iter().skip(k).all(|p| PARTICLES.contains(p))
|
||||
{
|
||||
let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default();
|
||||
return known(d, &base) || (base.contains('-') && hyphenated(d, &base));
|
||||
return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts));
|
||||
}
|
||||
match lp.as_slice() {
|
||||
// `кое-кто`, `кой-какой`.
|
||||
@@ -369,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
let last = parts.len() - 1;
|
||||
parts.iter().enumerate().all(|(i, p)| {
|
||||
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
|
||||
if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
|
||||
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
|
||||
return true;
|
||||
}
|
||||
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
|
||||
@@ -497,4 +461,25 @@ mod tests {
|
||||
crate::add_word("Зкщфрусскоеслово");
|
||||
assert!(is_correct("зкщфрусскоеслово"));
|
||||
}
|
||||
|
||||
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks
|
||||
/// those words against the dictionary instead of accepting them.
|
||||
#[test]
|
||||
fn proofing_options_reach_the_russian_check() {
|
||||
test_dict::install();
|
||||
let lang = crate::ProofLang::Ru;
|
||||
let defaults = crate::Options::default();
|
||||
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
|
||||
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
|
||||
assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default");
|
||||
assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps));
|
||||
assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps));
|
||||
assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too");
|
||||
assert!(crate::is_correct_in_with("превет2", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("превет2", lang, &check_digits));
|
||||
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
|
||||
}
|
||||
}
|
||||
|
||||
+70
-334
@@ -1,30 +1,14 @@
|
||||
//! Russian spelling suggestions, best first.
|
||||
//!
|
||||
//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the
|
||||
//! common confusions are defined here. Candidates come in rounds of rising cost, each checked
|
||||
//! against the dictionary:
|
||||
//! 1. letter case (`москва` → `Москва`);
|
||||
//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`,
|
||||
//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to
|
||||
//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`);
|
||||
//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or
|
||||
//! doubled consonant, `здраствуй` → `здравствуй`);
|
||||
//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped;
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word and share the
|
||||
//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within
|
||||
//! a letter group costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round.
|
||||
//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP`
|
||||
//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` →
|
||||
//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule
|
||||
//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning};
|
||||
|
||||
/// Letters confused with each other: a slip inside a group is cheaper than another typo.
|
||||
const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"];
|
||||
|
||||
/// Common confusions: a written sequence and what was meant.
|
||||
const REP: &[(&str, &str)] = &[
|
||||
("жы", "жи"),
|
||||
@@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[
|
||||
("ц", "тс"),
|
||||
("тс", "ц"),
|
||||
];
|
||||
|
||||
/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
"не",
|
||||
@@ -123,158 +106,42 @@ const PREPOSITIONS: &[&str] = &[
|
||||
const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"];
|
||||
/// The Russian alphabet, when the dictionary has no `TRY` line.
|
||||
const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё";
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 3;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
/// `е` written for `ё`, or the other way round, is barely a slip.
|
||||
const COST_YO: u32 = 3;
|
||||
const COST_MAP: u32 = 7;
|
||||
const COST_REP: u32 = 9;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`).
|
||||
const COST_SPLIT_PREPOSITION: u32 = 17;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
/// What Russian adds to its dictionary's tables.
|
||||
const TUNING: Tuning =
|
||||
Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT };
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance.
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The other members of each [`MAP`] group holding `c`.
|
||||
fn alternatives(c: char) -> Vec<char> {
|
||||
let mut out: Vec<char> = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect();
|
||||
out.sort_unstable();
|
||||
out.dedup();
|
||||
out
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip
|
||||
/// (`ё` is optional in ordinary writing).
|
||||
/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP }
|
||||
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP }
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
/// A missing space: cheap after a short leading word or before a short trailing one, dearer after
|
||||
/// a short preposition or between two longer words, never leaving a stray short half.
|
||||
fn split(a: &str, b: &str) -> Option<u32> {
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
|
||||
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
|
||||
let preposition = PREPOSITIONS.contains(&a);
|
||||
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
|
||||
return None;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
|
||||
Some(COST_SPLIT_PREPOSITION)
|
||||
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
Some(COST_SPLIT)
|
||||
} else if long(a) && long(b) {
|
||||
Some(COST_SPLIT_LONG)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's group representative (the first of its first group), lower case: `ё` → `е`,
|
||||
/// `з` → `з`, `с` → `з`.
|
||||
fn fold(c: char) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a slip within a letter group costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if fold(ai) == fold(bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
suggest_in(&super::dictionary(), word, max)
|
||||
@@ -282,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Slips within a letter group.
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in REP {
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
|
||||
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
|
||||
let preposition = PREPOSITIONS.contains(&a);
|
||||
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
|
||||
continue;
|
||||
}
|
||||
let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
|
||||
COST_SPLIT_PREPOSITION
|
||||
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = ALPHABET.chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
found.sort_by_key(|(_, k)| *k);
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a group relative of it,
|
||||
/// either case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(fold).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -458,6 +161,36 @@ mod tests {
|
||||
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
||||
}
|
||||
|
||||
/// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip
|
||||
/// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip;
|
||||
/// a preposition is never split off a verb, and after a one-letter one a typo is likelier.
|
||||
#[test]
|
||||
fn russian_costs_rank_its_slips() {
|
||||
let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap();
|
||||
assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]);
|
||||
assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]);
|
||||
assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into()));
|
||||
let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4);
|
||||
assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}");
|
||||
assert!(s.contains(&"в доме".into()), "{s:?}");
|
||||
}
|
||||
|
||||
/// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never
|
||||
/// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the
|
||||
/// shared Latin-script check doesn't reach it.
|
||||
#[test]
|
||||
fn nosuggest_words_are_never_offered() {
|
||||
let offered = |aff: &str, input: &str| {
|
||||
let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap();
|
||||
suggest_in(&d, input, 16)
|
||||
};
|
||||
assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared");
|
||||
for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] {
|
||||
let s = offered("SET UTF-8\nNOSUGGEST N\n", input);
|
||||
assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn russian_misspellings_get_the_right_suggestion() {
|
||||
crate::ru::test_dict::install();
|
||||
@@ -499,12 +232,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn distance_weighs_letter_groups_half() {
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
let map: Vec<Vec<char>> = MAP.iter().map(|g| g.chars().collect()).collect();
|
||||
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
||||
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
||||
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1);
|
||||
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3);
|
||||
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
//! The installed Hunspell dictionary of a proofing language, in one shape for every language:
|
||||
//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`].
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own.
|
||||
pub(crate) trait Dictionary: Default + Send + Sync {
|
||||
/// The installed `.aff` and `.dic` files, read.
|
||||
fn load(aff: &Path, dic: &Path) -> Result<Self, String>;
|
||||
/// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed.
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String>;
|
||||
/// No words: the language has no spelling marks.
|
||||
fn is_empty(&self) -> bool;
|
||||
}
|
||||
|
||||
impl Dictionary for AffixDict {
|
||||
fn load(aff: &Path, dic: &Path) -> Result<Self, String> {
|
||||
crate::hunspell::load(aff, dic)
|
||||
}
|
||||
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
|
||||
AffixDict::from_hunspell(aff, dic, &[])
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
AffixDict::is_empty(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// What every language's slot answers, whatever its dictionary type (the language table in
|
||||
/// lib.rs holds them side by side).
|
||||
pub(crate) trait Slot: Sync {
|
||||
/// See [`DictSlot::name`].
|
||||
fn name(&self) -> &'static str;
|
||||
/// See [`DictSlot::installed_path`].
|
||||
fn installed_path(&self) -> Option<PathBuf>;
|
||||
/// See [`DictSlot::has_dictionary`].
|
||||
fn has_dictionary(&self) -> bool;
|
||||
/// See [`DictSlot::reload`].
|
||||
fn reload(&self);
|
||||
}
|
||||
|
||||
impl<D: Dictionary> Slot for DictSlot<D> {
|
||||
fn name(&self) -> &'static str {
|
||||
DictSlot::name(self)
|
||||
}
|
||||
|
||||
fn installed_path(&self) -> Option<PathBuf> {
|
||||
DictSlot::installed_path(self)
|
||||
}
|
||||
|
||||
fn has_dictionary(&self) -> bool {
|
||||
DictSlot::has_dictionary(self)
|
||||
}
|
||||
|
||||
fn reload(&self) {
|
||||
DictSlot::reload(self);
|
||||
}
|
||||
}
|
||||
|
||||
/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on
|
||||
/// disk (it then has no file to report, even if a matching file also happens to exist): one lock,
|
||||
/// so a reload and an install can't interleave their two fields.
|
||||
struct State<D> {
|
||||
dict: Option<Arc<D>>,
|
||||
in_memory: bool,
|
||||
/// Counts reloads, so a load that started before one is not kept.
|
||||
reloads: u64,
|
||||
}
|
||||
|
||||
/// One language's dictionary.
|
||||
pub(crate) struct DictSlot<D> {
|
||||
/// File names the dictionary may have, the usual one first (also the key of its load error).
|
||||
names: &'static [&'static str],
|
||||
/// The language's English name, for the log.
|
||||
english: &'static str,
|
||||
state: RwLock<State<D>>,
|
||||
}
|
||||
|
||||
impl<D: Dictionary> DictSlot<D> {
|
||||
pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self {
|
||||
DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) }
|
||||
}
|
||||
|
||||
fn read(&self) -> RwLockReadGuard<'_, State<D>> {
|
||||
self.state.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn write(&self) -> RwLockWriteGuard<'_, State<D>> {
|
||||
self.state.write().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names.
|
||||
pub(crate) fn name(&self) -> &'static str {
|
||||
self.names.first().copied().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The file names the dictionary may have, the usual one first.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn names(&self) -> &'static [&'static str] {
|
||||
self.names
|
||||
}
|
||||
|
||||
/// The dictionary: the installed one (read on first use), one given with [`Self::install`], or
|
||||
/// an empty one when none is available (the language's spelling is off then).
|
||||
pub(crate) fn get(&self) -> Arc<D> {
|
||||
let reloads = {
|
||||
let s = self.read();
|
||||
if let Some(d) = s.dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
s.reloads
|
||||
};
|
||||
self.keep(reloads, Arc::new(self.load_installed()))
|
||||
}
|
||||
|
||||
/// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the
|
||||
/// load may have read the old folder: it serves this one check and the next one loads again.
|
||||
fn keep(&self, reloads: u64, loaded: Arc<D>) -> Arc<D> {
|
||||
let mut s = self.write();
|
||||
if s.reloads != reloads {
|
||||
return s.dict.clone().unwrap_or(loaded);
|
||||
}
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
s.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed(&self) -> D {
|
||||
let name = self.name();
|
||||
let Some((aff, dic)) = crate::hunspell::find(self.names) else {
|
||||
crate::hunspell::clear_load_error(name);
|
||||
log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english);
|
||||
return D::default();
|
||||
};
|
||||
crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("{} spelling: {e}", self.english);
|
||||
D::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for
|
||||
/// example one the user picked. On error the current dictionary stays.
|
||||
pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = D::parse(aff, dic)?;
|
||||
let mut slot = self.write();
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub(crate) fn reload(&self) {
|
||||
let mut slot = self.write();
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
slot.reloads = slot.reloads.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed dictionary, searching the dictionaries folder and the
|
||||
/// system locations now (File › Options › Proofing shows it); `None` when there is none, or
|
||||
/// when the dictionary in use was given with [`Self::install`] and so has no file.
|
||||
pub(crate) fn installed_path(&self) -> Option<PathBuf> {
|
||||
if self.read().in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(self.names).map(|(_, dic)| dic)
|
||||
}
|
||||
|
||||
/// Is a dictionary available (spelling is checked)?
|
||||
pub(crate) fn has_dictionary(&self) -> bool {
|
||||
!self.get().is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
static SLOT: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_TEST"], "Test");
|
||||
|
||||
#[test]
|
||||
fn an_installed_dictionary_has_no_file_until_reloaded() {
|
||||
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
|
||||
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
|
||||
SLOT.install("", "1\nkot\n").unwrap();
|
||||
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
|
||||
assert_eq!(SLOT.installed_path(), None);
|
||||
SLOT.reload();
|
||||
assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none");
|
||||
assert_eq!(SLOT.name(), "zz_SLOT_TEST");
|
||||
}
|
||||
|
||||
/// A load that started before a dictionaries-folder change is not kept after its reload.
|
||||
#[test]
|
||||
fn a_load_from_before_a_reload_is_not_kept() {
|
||||
let slot: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test");
|
||||
let before = slot.read().reloads;
|
||||
slot.reload();
|
||||
let _ = slot.keep(before, Arc::new(AffixDict::default()));
|
||||
assert!(slot.read().dict.is_none(), "the next check loads from the new folder");
|
||||
let now = slot.read().reloads;
|
||||
let _ = slot.keep(now, Arc::new(AffixDict::default()));
|
||||
assert!(slot.read().dict.is_some(), "a load since the reload is kept");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian,
|
||||
//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the
|
||||
//! File › Options › Proofing choices. Each language goes on from there with its own rules.
|
||||
|
||||
use crate::Options;
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one these dictionaries cover)?
|
||||
fn latin(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it
|
||||
/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single
|
||||
/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the
|
||||
/// options skip those, or it is in another script.
|
||||
pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> {
|
||||
if d.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return None;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return None;
|
||||
}
|
||||
w.chars().any(latin).then_some(w)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn words_needing_no_lookup() {
|
||||
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
|
||||
let defaults = Options::default();
|
||||
assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary");
|
||||
for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] {
|
||||
assert_eq!(word_to_check(&d, w, &defaults), None, "{w}");
|
||||
}
|
||||
assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot"));
|
||||
assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot"));
|
||||
let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults };
|
||||
assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO"));
|
||||
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
|
||||
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,20 @@ pub struct TrDict {
|
||||
max_suffix_len: usize,
|
||||
}
|
||||
|
||||
impl crate::slot::Dictionary for TrDict {
|
||||
fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
|
||||
super::load(aff, dic)
|
||||
}
|
||||
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
|
||||
TrDict::from_hunspell(aff, dic)
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
TrDict::is_empty(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl TrDict {
|
||||
/// Returns true if empty (no dictionary loaded).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
|
||||
+43
-46
@@ -15,39 +15,26 @@ pub mod dict;
|
||||
pub mod grammar;
|
||||
pub mod hyphen;
|
||||
pub mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use dict::TrDict;
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<TrDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Turkish Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"];
|
||||
|
||||
/// The Turkish dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<TrDict> = DictSlot::new(NAMES, "Turkish");
|
||||
|
||||
/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available.
|
||||
pub fn dictionary() -> Arc<TrDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> TrDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed");
|
||||
return TrDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Turkish spelling: {e}");
|
||||
TrDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
|
||||
@@ -57,28 +44,15 @@ pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDic
|
||||
TrDict::from_hunspell(&aff_text, &dic_text)
|
||||
}
|
||||
|
||||
/// Use this Turkish dictionary from now on.
|
||||
/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
|
||||
/// current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = TrDict::from_hunspell(aff, dic)?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reload the dictionary on next check.
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Turkish dictionary available?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Turkish?
|
||||
@@ -86,15 +60,26 @@ pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain
|
||||
/// numbers off, words with digits are checked against the dictionary too.
|
||||
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
|
||||
check(&dictionary(), word, opts)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary.
|
||||
pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
|
||||
check(d, word, &crate::Options::default())
|
||||
}
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // No Turkish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
// Check user dictionary first
|
||||
@@ -119,7 +104,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
|
||||
}
|
||||
|
||||
// Compound words separated by hyphens (e.g., `afyon-karahisar`)
|
||||
if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) {
|
||||
if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -145,11 +130,8 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn installed_dictionary_checks_words_if_present() {
|
||||
// Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check.
|
||||
if !has_dictionary() {
|
||||
return;
|
||||
}
|
||||
fn installed_dictionary_checks_words() {
|
||||
test_dict::install();
|
||||
assert!(is_correct("kitap"));
|
||||
assert!(is_correct("kitaplar"));
|
||||
assert!(is_correct("evde"));
|
||||
@@ -164,4 +146,19 @@ mod tests {
|
||||
crate::add_word("benimbilmedigimkelime");
|
||||
assert!(is_correct("benimbilmedigimkelime"));
|
||||
}
|
||||
|
||||
/// With Ignore words that contain numbers off, Turkish checks words with digits against the
|
||||
/// dictionary instead of accepting them, hyphen parts included.
|
||||
#[test]
|
||||
fn proofing_options_reach_the_turkish_check() {
|
||||
test_dict::install();
|
||||
let lang = crate::ProofLang::Tr;
|
||||
let defaults = crate::Options::default();
|
||||
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
|
||||
assert!(crate::is_correct_in_with("kitapp2", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits));
|
||||
assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits));
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,10 +318,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn typo_transposition_is_suggested() {
|
||||
// Suggestions come from the installed Turkish dictionary; without one there are none.
|
||||
if !super::super::has_dictionary() {
|
||||
return;
|
||||
}
|
||||
crate::tr::test_dict::install();
|
||||
let sugs = suggest("sorna", 3);
|
||||
assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}");
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from
|
||||
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
|
||||
|
||||
/// No affix rules: the forms the tests look up are listed whole.
|
||||
pub const AFF: &str = "SET UTF-8\n";
|
||||
|
||||
/// The words the Turkish tests expect to be known.
|
||||
pub const DIC: &str = "7
|
||||
kitap
|
||||
kitaplar
|
||||
ev
|
||||
evde
|
||||
olacaklardan
|
||||
sonra
|
||||
geldin
|
||||
";
|
||||
|
||||
pub fn install() {
|
||||
if let Err(e) = super::install_dictionary(AFF, DIC) {
|
||||
panic!("test dictionary: {e}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user