diff --git a/Cargo.lock b/Cargo.lock index 91a380735..bf71c7d9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4017,6 +4017,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.53.2" @@ -4197,6 +4203,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -5379,6 +5394,7 @@ dependencies = [ "log", "miniz_oxide 0.8.9", "proptest", + "unicode-normalization", ] [[package]] diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index 779cef12b..2cad4f19f 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -1185,12 +1185,12 @@ impl Session { } /// The Spanish of the region in effect (`es-MX` on a Mexican system, `es-UY` on a Uruguayan - /// one), or Spain's (`es-ES`) when the region's language isn't Spanish or it names no - /// territory. A Spanish interface writes new documents in it, so they are proofed in the + /// one; the ISO 639-2 `spa` counts as Spanish), or Spain's (`es-ES`) when the region's + /// language isn't Spanish or it names no territory. A Spanish interface writes new documents in it, so they are proofed in the /// writer's Spanish from the first word (#3360). pub fn region_spanish(&self) -> String { let r = self.region(); - (r.language == "es" && !r.territory.is_empty()) + (matches!(r.language.as_str(), "es" | "spa") && !r.territory.is_empty()) .then(|| format!("es-{}", r.territory)) .and_then(|t| wordcraft_proof::lang::normalize_tag(&t)) .unwrap_or_else(|| "es-ES".into()) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 18277b1cf..74525308f 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -7026,6 +7026,10 @@ fn new_documents_follow_the_regions_spanish() { let mut s = s(); s.system_region = wordcraft_geom::Region::from_tag("es_UY.UTF-8"); assert_eq!(s.region_spanish(), "es-UY"); + // The ISO 639-2 code is Spanish too (CodeRabbit review on #3562). + s.system_region = wordcraft_geom::Region::from_tag("spa_MX.UTF-8"); + assert_eq!(s.region_spanish(), "es-MX"); + s.system_region = wordcraft_geom::Region::from_tag("es_UY.UTF-8"); s.set_editing_language(Some(&s.region_spanish())); assert_eq!(s.doc.styles.default_chr.lang.as_deref(), Some("es-UY"), "the untouched document follows"); for (tag, spanish) in [("es-MX", "es-MX"), ("es-419", "es-419"), ("es-ES", "es-ES"), ("en-US", "es-ES"), ("fr-FR", "es-ES"), ("auto", "es-UY")] { diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index abee19074..234684a9f 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -11,6 +11,7 @@ workspace = true [dependencies] log = { workspace = true } miniz_oxide = "0.8" +unicode-normalization = "0.1" [dev-dependencies] proptest = { workspace = true } diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index a22a6ff87..35aaa50d0 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -6,6 +6,8 @@ //! dices?`), and `(?)` and `(!)`, which mark doubt or surprise, stand alone. No other //! Spanish-specific grammar, comma or sentence-capital rules are inferred. +use unicode_normalization::UnicodeNormalization; + use crate::{Issue, IssueKind, ProofLang, words}; fn issue(start: usize, end: usize, message: String, suggestions: Vec, lang: ProofLang) -> Issue { @@ -27,7 +29,11 @@ fn repeated_words(text: &str, ws: &[(usize, usize)], lang: ProofLang, v: &mut Ve for pair in ws.windows(2) { let [(a0, b0), (a1, b1)] = pair else { continue }; let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; - if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + // Letters and the combining marks of a decomposed word (`tambie\u{301}n`); a composed and + // a decomposed spelling of one word are the same word. + let letters = |w: &str| w.chars().all(|c| c.is_alphabetic() || unicode_normalization::char::is_combining_mark(c)); + let folded = |w: &str| w.nfc().collect::().to_lowercase(); + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && letters(w0) && folded(w0) == folded(w1) { v.push(issue(*a0, *b1, format!("Palabra repetida: «{w1}»"), vec![w0.to_string()], lang)); } } @@ -83,8 +89,33 @@ fn is_boundary(c: char) -> bool { matches!(c, '.' | '?' | '!' | '…' | ':' | ';' | '\n' | '\r' | '\t' | '«' | '“' | '"' | '(' | '[' | '—') } -/// A closing `?` or `!` whose sentence has no opening `¿` or `¡`. +/// The byte ranges of the whitespace-delimited tokens of `text` that are web or e-mail addresses +/// (`https://example.com/?`, `www.example.com`, `ana@example.com`), as spelling skips them. +fn address_ranges(text: &str) -> Vec<(usize, usize)> { + let mut out = Vec::new(); + let mut start = None; + // A final space ends the last token. + for (i, c) in text.char_indices().chain(std::iter::once((text.len(), ' '))) { + if !c.is_whitespace() { + start = start.or(Some(i)); + } else if let Some(s) = start.take() + && text.get(s..i).is_some_and(crate::is_address) + { + out.push((s, i)); + } + } + out +} + +/// A closing `?` or `!` whose sentence has no opening `¿` or `¡`. Marks in web and e-mail +/// addresses aren't sentence punctuation, and neither is punctuation inside an address or a word +/// (`example.com`, `3.5`, `10:30`) a sentence's start. fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec) { + let addresses = address_ranges(text); + let in_address = |j: usize| addresses.iter().any(|&(s, e)| s <= j && j < e); + // An address's own punctuation starts no sentence, except a mark that ends it + // (`…/? Vienes mañana?`): that one isn't flagged, but whatever follows is a new sentence. + let ends_address = |j: usize, b: char| addresses.iter().any(|&(_, e)| j + b.len_utf8() == e) && matches!(b, '?' | '!' | '.'); for (i, c) in text.char_indices() { let (opener, message) = match c { '?' => ('¿', "Falta el signo de interrogación de apertura (¿)"), @@ -94,12 +125,15 @@ fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec) { let before = text.get(..i).unwrap_or(""); let prev = before.chars().next_back(); let next = text.get(i + 1..).and_then(|r| r.chars().next()); - // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=` and `a!b` - // aren't sentence ends. - if matches!(prev, Some('?' | '!')) || (prev == Some('(') && next == Some(')')) || next.is_some_and(char::is_alphanumeric) { + // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=`, `a!b` and a + // mark ending an address (`https://example.com/?`) aren't sentence ends. + if matches!(prev, Some('?' | '!')) || (prev == Some('(') && next == Some(')')) || next.is_some_and(char::is_alphanumeric) || in_address(i) { continue; } - let from = before.char_indices().rev().find(|&(_, b)| is_boundary(b) || matches!(b, '¿' | '¡')); + let within_word = |j: usize, b: char| text.get(j + b.len_utf8()..).and_then(|r| r.chars().next()).is_some_and(char::is_alphanumeric); + let from = before.char_indices().rev().find(|&(j, b)| { + matches!(b, '¿' | '¡') || ends_address(j, b) || (is_boundary(b) && !in_address(j) && !(matches!(b, '.' | ':' | ';') && within_word(j, b))) + }); if from.is_some_and(|(_, b)| matches!(b, '¿' | '¡')) { continue; } @@ -165,6 +199,47 @@ mod tests { } } + /// Maintainer review on #3562: a URL ending in `/?` isn't a question, and the dots and colons + /// of an address or a word (`example.com`, `https:`, `10:30`) don't start the sentence the + /// missing `¿` is offered for; ordinary questions are still checked. + #[test] + fn addresses_are_not_questions_or_sentence_starts() { + let missing = |text: &str| -> Vec<(String, Vec)> { + check(text, ProofLang::Es) + .into_iter() + .filter(|i| i.message.starts_with("Falta")) + .map(|i| (text[i.start..i.end].to_string(), i.suggestions)) + .collect() + }; + for text in [ + "Ver https://example.com/?", + "Ver https://example.com/? ¿Vienes?", + "Ver http://example.com/buscar/? y www.example.com/?", + "Escribí a ana@example.com!", + "¿Viste https://example.com/?", + ] { + assert!(missing(text).is_empty(), "{text}: {:?}", missing(text)); + } + assert_eq!( + missing("Mirá https://example.com/ y decime si vienes?"), + [("Mirá https://example.com/ y decime si vienes?".to_string(), vec!["¿Mirá https://example.com/ y decime si vienes?".to_string()])] + ); + assert_eq!(missing("Mirá example.com y decime si vienes?")[0].0, "Mirá example.com y decime si vienes?"); + assert_eq!(missing("Son las 10:30, vienes?")[0].0, "Son las 10:30, vienes?"); + assert_eq!(missing("Ver https://example.com/? Vienes mañana?"), [("Vienes mañana?".to_string(), vec!["¿Vienes mañana?".to_string()])]); + assert_eq!(missing("Llegó. Vienes?")[0].0, "Vienes?", "a full stop before a space still ends a sentence"); + } + + /// A word repeated once composed and once decomposed is still repeated. + #[test] + fn repeated_words_match_across_normalization_forms() { + let text = "también tambie\u{301}n"; + let issues = check(text, ProofLang::Es); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!((issues[0].start, issues[0].end), (0, text.len())); + assert!(issues[0].message.starts_with("Palabra repetida")); + } + #[test] fn a_space_after_an_opening_mark_is_flagged() { let text = "¿ Vienes? ¡ Hola!"; diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index b18b4f46a..46d7ab371 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -304,6 +304,25 @@ mod tests { assert!(is_correct("PRUEVA", ProofLang::Es), "direct checks keep the defaults"); } + /// Maintainer review on #3562: a word typed decomposed (NFD: `e` + U+0301) is checked like its + /// composed form (NFC) and keeps its own byte range, accents and all. + #[test] + fn decomposed_accents_are_checked_like_composed_ones() { + test_dict::install(); + for lang in VARIETIES { + for text in ["esto también", "esto tambie\u{301}n", "una cancio\u{301}n y pingu\u{308}ino", "CANCIO\u{301}N", "nin\u{303}o"] { + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + assert!(issues.is_empty(), "{lang:?} {text:?}: {issues:?}"); + } + // A misspelling that ends in a decomposed accent is flagged whole, at its own bytes. + let text = "esto es una prueva tambie\u{301}nn"; + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + let flagged: Vec<(&str, usize, usize)> = issues.iter().map(|i| (&text[i.start..i.end], i.start, i.end)).collect(); + assert_eq!(flagged, [("prueva", 12, 18), ("tambie\u{301}nn", 19, text.len())], "{lang:?}"); + assert!(crate::suggest_in("tambie\u{301}nn", 8, lang).contains(&"también".into()), "{lang:?}"); + } + } + #[test] fn forbidden_words_are_rejected_in_every_case() { test_dict::install(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 4e2c3f2fe..cb70939d4 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -626,11 +626,12 @@ fn damerau(a: &[char], b: &str) -> usize { prev.get(m).copied().unwrap_or(9) } -/// Words in `text`: (byte start, byte end). +/// Words in `text`: (byte start, byte end). Combining marks belong to the word they follow, so a +/// word typed decomposed (`tambie\u{301}n`, NFD) is one word, as its precomposed form is. pub fn words(text: &str) -> Vec<(usize, usize)> { let mut out = Vec::new(); let mut start: Option = None; - let is_w = |c: char| c.is_alphanumeric() || c == '\'' || c == '’' || c == '-'; + let is_w = |c: char| c.is_alphanumeric() || c == '\'' || c == '’' || c == '-' || unicode_normalization::char::is_combining_mark(c); for (i, c) in text.char_indices() { match (start, is_w(c)) { (None, true) => start = Some(i), @@ -698,8 +699,13 @@ fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0); let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); - let token = text.get(ts..te).unwrap_or(""); - (ts, te, token.contains("://") || token.contains('@') || token.starts_with("www.")) + (ts, te, text.get(ts..te).is_some_and(is_address)) +} + +/// Does a whitespace-delimited token look like a web or e-mail address (`https://…`, `www.…`, +/// `a@b.org`)? Spelling skips its words, and Spanish grammar its punctuation. +pub(crate) fn is_address(token: &str) -> bool { + token.contains("://") || token.contains('@') || token.starts_with("www.") } /// The spelling issue of the word at `a..b` of `text`, if it is misspelled in `lang`. Words in @@ -731,8 +737,21 @@ pub fn is_correct_in(word: &str, lang: ProofLang) -> bool { is_correct_in_with(word, lang, &Options::default()) } -/// [`is_correct_in`] with the File › Options › Proofing choices. +/// `word` composed (NFC), as the dictionaries spell their words: `tambie\u{301}n` is looked up as +/// `también`. Words already composed (all ASCII ones among them) are borrowed as they are. +fn composed(word: &str) -> std::borrow::Cow<'_, str> { + use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick}; + if word.is_ascii() || is_nfc_quick(word.chars()) == IsNormalized::Yes { + std::borrow::Cow::Borrowed(word) + } else { + std::borrow::Cow::Owned(word.nfc().collect()) + } +} + +/// [`is_correct_in`] with the File › Options › Proofing choices. A word with combining marks is +/// checked as its composed form ([`composed`]). pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { + let word = &*composed(word); match lang { ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts), ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct(word), @@ -750,8 +769,10 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { } } -/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`). +/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed +/// ([`composed`]) like the dictionaries' words. pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { + let word = &*composed(word); match lang { ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang), ProofLang::Pl => pl::suggest(word, max), @@ -1617,6 +1638,20 @@ mod tests { } } + /// Combining marks stay in their word (#3562 review): a decomposed word is one word, with the + /// byte range of what was typed, and it is checked as its composed form in every language. + #[test] + fn decomposed_words_stay_whole() { + let text = "tambie\u{301}n, cafe\u{301} \u{301}x"; + let ws: Vec<&str> = words(text).into_iter().map(|(a, b)| &text[a..b]).collect(); + assert_eq!(ws, ["tambie\u{301}n", "cafe\u{301}", "\u{301}x"]); + assert_eq!(words("tambie\u{301}n"), [(0, 9)]); + assert_eq!(&*composed("tambie\u{301}n"), "también"); + assert!(matches!(composed("también"), std::borrow::Cow::Borrowed(_)) && matches!(composed("plain"), std::borrow::Cow::Borrowed(_))); + pt::test_dict::install(); + assert!(is_correct_in("corac\u{327}a\u{303}o", ProofLang::Pt), "Portuguese gets the same composition"); + } + /// #100: runs in a language without a dictionary are never flagged; English still is. #[test] fn non_english_words_are_not_flagged() {