Decode ISO 8859-1 Hunspell dictionaries so the original pt_BR works (#2824) (#2904)

Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
MarekSpodBiedry
2026-10-18 11:50:59 -04:00
committed by GitHub
co-authored by Brandon Thomas
parent 03a992a46a
commit 69302481ad
2 changed files with 21 additions and 5 deletions
+17 -2
View File
@@ -128,6 +128,11 @@ pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
}
/// ISO 8859-1: every byte is the code point of the same value (the original VERO pt_BR uses it).
fn decode_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&b| char::from(b)).collect()
}
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
@@ -152,8 +157,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
}
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, KOI8-R and
/// Windows-1251 the ones older Russian dictionaries use).
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
@@ -162,6 +167,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
@@ -185,6 +191,15 @@ mod tests {
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
}
#[test]
fn decodes_latin1() {
// `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1.
let (a, d) = decode(b"SET ISO8859-1\nTRY \xe7\n", b"1\nor\xe7amento/A\nesper\xe1vamos\n").unwrap();
assert!(a.contains("TRY ç"), "{a}");
assert_eq!(d, "1\norçamento/A\nesperávamos\n");
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
+4 -3
View File
@@ -1,6 +1,6 @@
# Installing spelling dictionaries
> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-16 · **Change:** major (first version)
> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-18 · **Change:** trivial (ISO 8859-1 dictionaries are read, #2824; previously major: first version)
English spelling works out of the box. Polish, Russian, Portuguese, Czech, Italian, Croatian and
Turkish spelling use a standard **Hunspell** dictionary (an `.aff` and a `.dic` file) that
@@ -28,8 +28,8 @@ WordCraft looks for the first of these file names (each name means a `<name>.aff
Every Portuguese region uses the same `pt_BR` dictionary, so text marked *Português (Portugal)*
is checked with it too.
The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-2, KOI8-R and
Windows-1251 are read as well).
The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-1, ISO 8859-2, KOI8-R
and Windows-1251 are read as well; the original VERO `pt_BR` is ISO 8859-1).
## Where WordCraft looks
@@ -122,4 +122,5 @@ Never commit dictionary files to this repository: they stay on the user's system
| Date | Change | Summary |
|---|---|---|
| 2026-10-18 | trivial | ISO 8859-1 (`SET ISO8859-1`) dictionaries are read, so the original VERO `pt_BR` works without converting it to UTF-8 (#2824) |
| 2026-10-16 | major | First version: the file names per language, the search order and folders on Windows, macOS and Linux, installing `pt_BR` on each, and how to check it works |