#3360: decomposed accents stay in their word; URLs aren't Spanish questions

Maintainer review on #3562:
- words() keeps combining marks with the word they follow, so a word typed
  decomposed (NFD, `tambie\u{301}n`) is one word with its own byte range;
  is_correct_in_with and suggest_in look it up composed (NFC), so it is
  checked like `también` in every language (unicode-normalization, MIT/Apache)
- the Spanish missing-¿/¡ rule skips marks in web and e-mail addresses
  (`Ver https://example.com/?`), and punctuation inside an address or a word
  (`example.com`, `https:`, `10:30`) no longer starts the sentence the
  opening mark is offered for; a mark ending an address still ends a sentence
- Spanish repeated-word check matches composed and decomposed spellings
- regressions: check_text with the Spanish fixture and NFC/NFD equivalents
  (acceptance and ranges), words() and composed(), URLs ending in /? beside
  ordinary questions

CodeRabbit review: Session::region_spanish takes the ISO 639-2 `spa` as
Spanish, so a `spa-MX` region writes es-MX documents.

Cargo.lock

@@ -4017,6 +4017,12 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinyvec"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
+
 [[package]]
 name = "tokio"
 version = "1.53.2"
@@ -4197,6 +4203,15 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
 
+[[package]]
+name = "unicode-normalization"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "unicode-properties"
 version = "0.1.4"
@@ -5379,6 +5394,7 @@ dependencies = [
  "log",
  "miniz_oxide 0.8.9",
  "proptest",
+ "unicode-normalization",
 ]
 
 [[package]]

crates/engine/src/session.rs

@@ -1185,12 +1185,12 @@ impl Session {
     }
 
     /// The Spanish of the region in effect (`es-MX` on a Mexican system, `es-UY` on a Uruguayan
-    /// one), or Spain's (`es-ES`) when the region's language isn't Spanish or it names no
-    /// territory. A Spanish interface writes new documents in it, so they are proofed in the
+    /// one; the ISO 639-2 `spa` counts as Spanish), or Spain's (`es-ES`) when the region's
+    /// language isn't Spanish or it names no territory. A Spanish interface writes new documents in it, so they are proofed in the
     /// writer's Spanish from the first word (#3360).
     pub fn region_spanish(&self) -> String {
         let r = self.region();
-        (r.language == "es" && !r.territory.is_empty())
+        (matches!(r.language.as_str(), "es" | "spa") && !r.territory.is_empty())
             .then(|| format!("es-{}", r.territory))
             .and_then(|t| wordcraft_proof::lang::normalize_tag(&t))
             .unwrap_or_else(|| "es-ES".into())

crates/engine/src/tests.rs

@@ -7026,6 +7026,10 @@ fn new_documents_follow_the_regions_spanish() {
     let mut s = s();
     s.system_region = wordcraft_geom::Region::from_tag("es_UY.UTF-8");
     assert_eq!(s.region_spanish(), "es-UY");
+    // The ISO 639-2 code is Spanish too (CodeRabbit review on #3562).
+    s.system_region = wordcraft_geom::Region::from_tag("spa_MX.UTF-8");
+    assert_eq!(s.region_spanish(), "es-MX");
+    s.system_region = wordcraft_geom::Region::from_tag("es_UY.UTF-8");
     s.set_editing_language(Some(&s.region_spanish()));
     assert_eq!(s.doc.styles.default_chr.lang.as_deref(), Some("es-UY"), "the untouched document follows");
     for (tag, spanish) in [("es-MX", "es-MX"), ("es-419", "es-419"), ("es-ES", "es-ES"), ("en-US", "es-ES"), ("fr-FR", "es-ES"), ("auto", "es-UY")] {

crates/proof/Cargo.toml

@@ -11,6 +11,7 @@ workspace = true
 [dependencies]
 log = { workspace = true }
 miniz_oxide = "0.8"
+unicode-normalization = "0.1"
 
 [dev-dependencies]
 proptest = { workspace = true }

crates/proof/src/es/grammar.rs

@@ -6,6 +6,8 @@
 //! dices?`), and `(?)` and `(!)`, which mark doubt or surprise, stand alone. No other
 //! Spanish-specific grammar, comma or sentence-capital rules are inferred.
 
+use unicode_normalization::UnicodeNormalization;
+
 use crate::{Issue, IssueKind, ProofLang, words};
 
 fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>, lang: ProofLang) -> Issue {
@@ -27,7 +29,11 @@ fn repeated_words(text: &str, ws: &[(usize, usize)], lang: ProofLang, v: &mut Ve
     for pair in ws.windows(2) {
         let [(a0, b0), (a1, b1)] = pair else { continue };
         let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
-        if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() {
+        // Letters and the combining marks of a decomposed word (`tambie\u{301}n`); a composed and
+        // a decomposed spelling of one word are the same word.
+        let letters = |w: &str| w.chars().all(|c| c.is_alphabetic() || unicode_normalization::char::is_combining_mark(c));
+        let folded = |w: &str| w.nfc().collect::<String>().to_lowercase();
+        if !gap.is_empty() && gap.chars().all(char::is_whitespace) && letters(w0) && folded(w0) == folded(w1) {
             v.push(issue(*a0, *b1, format!("Palabra repetida: «{w1}»"), vec![w0.to_string()], lang));
         }
     }
@@ -83,8 +89,33 @@ fn is_boundary(c: char) -> bool {
     matches!(c, '.' | '?' | '!' | '…' | ':' | ';' | '\n' | '\r' | '\t' | '«' | '“' | '"' | '(' | '[' | '—')
 }
 
-/// A closing `?` or `!` whose sentence has no opening `¿` or `¡`.
+/// The byte ranges of the whitespace-delimited tokens of `text` that are web or e-mail addresses
+/// (`https://example.com/?`, `www.example.com`, `ana@example.com`), as spelling skips them.
+fn address_ranges(text: &str) -> Vec<(usize, usize)> {
+    let mut out = Vec::new();
+    let mut start = None;
+    // A final space ends the last token.
+    for (i, c) in text.char_indices().chain(std::iter::once((text.len(), ' '))) {
+        if !c.is_whitespace() {
+            start = start.or(Some(i));
+        } else if let Some(s) = start.take()
+            && text.get(s..i).is_some_and(crate::is_address)
+        {
+            out.push((s, i));
+        }
+    }
+    out
+}
+
+/// A closing `?` or `!` whose sentence has no opening `¿` or `¡`. Marks in web and e-mail
+/// addresses aren't sentence punctuation, and neither is punctuation inside an address or a word
+/// (`example.com`, `3.5`, `10:30`) a sentence's start.
 fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
+    let addresses = address_ranges(text);
+    let in_address = |j: usize| addresses.iter().any(|&(s, e)| s <= j && j < e);
+    // An address's own punctuation starts no sentence, except a mark that ends it
+    // (`…/? Vienes mañana?`): that one isn't flagged, but whatever follows is a new sentence.
+    let ends_address = |j: usize, b: char| addresses.iter().any(|&(_, e)| j + b.len_utf8() == e) && matches!(b, '?' | '!' | '.');
     for (i, c) in text.char_indices() {
         let (opener, message) = match c {
             '?' => ('¿', "Falta el signo de interrogación de apertura (¿)"),
@@ -94,12 +125,15 @@ fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
         let before = text.get(..i).unwrap_or("");
         let prev = before.chars().next_back();
         let next = text.get(i + 1..).and_then(|r| r.chars().next());
-        // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=` and `a!b`
-        // aren't sentence ends.
-        if matches!(prev, Some('?' | '!')) || (prev == Some('(') && next == Some(')')) || next.is_some_and(char::is_alphanumeric) {
+        // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=`, `a!b` and a
+        // mark ending an address (`https://example.com/?`) aren't sentence ends.
+        if matches!(prev, Some('?' | '!')) || (prev == Some('(') && next == Some(')')) || next.is_some_and(char::is_alphanumeric) || in_address(i) {
             continue;
         }
-        let from = before.char_indices().rev().find(|&(_, b)| is_boundary(b) || matches!(b, '¿' | '¡'));
+        let within_word = |j: usize, b: char| text.get(j + b.len_utf8()..).and_then(|r| r.chars().next()).is_some_and(char::is_alphanumeric);
+        let from = before.char_indices().rev().find(|&(j, b)| {
+            matches!(b, '¿' | '¡') || ends_address(j, b) || (is_boundary(b) && !in_address(j) && !(matches!(b, '.' | ':' | ';') && within_word(j, b)))
+        });
         if from.is_some_and(|(_, b)| matches!(b, '¿' | '¡')) {
             continue;
         }
@@ -165,6 +199,47 @@ mod tests {
         }
     }
 
+    /// Maintainer review on #3562: a URL ending in `/?` isn't a question, and the dots and colons
+    /// of an address or a word (`example.com`, `https:`, `10:30`) don't start the sentence the
+    /// missing `¿` is offered for; ordinary questions are still checked.
+    #[test]
+    fn addresses_are_not_questions_or_sentence_starts() {
+        let missing = |text: &str| -> Vec<(String, Vec<String>)> {
+            check(text, ProofLang::Es)
+                .into_iter()
+                .filter(|i| i.message.starts_with("Falta"))
+                .map(|i| (text[i.start..i.end].to_string(), i.suggestions))
+                .collect()
+        };
+        for text in [
+            "Ver https://example.com/?",
+            "Ver https://example.com/? ¿Vienes?",
+            "Ver http://example.com/buscar/? y www.example.com/?",
+            "Escribí a ana@example.com!",
+            "¿Viste https://example.com/?",
+        ] {
+            assert!(missing(text).is_empty(), "{text}: {:?}", missing(text));
+        }
+        assert_eq!(
+            missing("Mirá https://example.com/ y decime si vienes?"),
+            [("Mirá https://example.com/ y decime si vienes?".to_string(), vec!["¿Mirá https://example.com/ y decime si vienes?".to_string()])]
+        );
+        assert_eq!(missing("Mirá example.com y decime si vienes?")[0].0, "Mirá example.com y decime si vienes?");
+        assert_eq!(missing("Son las 10:30, vienes?")[0].0, "Son las 10:30, vienes?");
+        assert_eq!(missing("Ver https://example.com/? Vienes mañana?"), [("Vienes mañana?".to_string(), vec!["¿Vienes mañana?".to_string()])]);
+        assert_eq!(missing("Llegó. Vienes?")[0].0, "Vienes?", "a full stop before a space still ends a sentence");
+    }
+
+    /// A word repeated once composed and once decomposed is still repeated.
+    #[test]
+    fn repeated_words_match_across_normalization_forms() {
+        let text = "también tambie\u{301}n";
+        let issues = check(text, ProofLang::Es);
+        assert_eq!(issues.len(), 1, "{issues:?}");
+        assert_eq!((issues[0].start, issues[0].end), (0, text.len()));
+        assert!(issues[0].message.starts_with("Palabra repetida"));
+    }
+
     #[test]
     fn a_space_after_an_opening_mark_is_flagged() {
         let text = "¿ Vienes? ¡ Hola!";

crates/proof/src/es/mod.rs

@@ -304,6 +304,25 @@ mod tests {
         assert!(is_correct("PRUEVA", ProofLang::Es), "direct checks keep the defaults");
     }
 
+    /// Maintainer review on #3562: a word typed decomposed (NFD: `e` + U+0301) is checked like its
+    /// composed form (NFC) and keeps its own byte range, accents and all.
+    #[test]
+    fn decomposed_accents_are_checked_like_composed_ones() {
+        test_dict::install();
+        for lang in VARIETIES {
+            for text in ["esto también", "esto tambie\u{301}n", "una cancio\u{301}n y pingu\u{308}ino", "CANCIO\u{301}N", "nin\u{303}o"] {
+                let issues = crate::check_text(text, &[(0..text.len(), lang)]);
+                assert!(issues.is_empty(), "{lang:?} {text:?}: {issues:?}");
+            }
+            // A misspelling that ends in a decomposed accent is flagged whole, at its own bytes.
+            let text = "esto es una prueva tambie\u{301}nn";
+            let issues = crate::check_text(text, &[(0..text.len(), lang)]);
+            let flagged: Vec<(&str, usize, usize)> = issues.iter().map(|i| (&text[i.start..i.end], i.start, i.end)).collect();
+            assert_eq!(flagged, [("prueva", 12, 18), ("tambie\u{301}nn", 19, text.len())], "{lang:?}");
+            assert!(crate::suggest_in("tambie\u{301}nn", 8, lang).contains(&"también".into()), "{lang:?}");
+        }
+    }
+
     #[test]
     fn forbidden_words_are_rejected_in_every_case() {
         test_dict::install();

crates/proof/src/lib.rs

@@ -626,11 +626,12 @@ fn damerau(a: &[char], b: &str) -> usize {
     prev.get(m).copied().unwrap_or(9)
 }
 
-/// Words in `text`: (byte start, byte end).
+/// Words in `text`: (byte start, byte end). Combining marks belong to the word they follow, so a
+/// word typed decomposed (`tambie\u{301}n`, NFD) is one word, as its precomposed form is.
 pub fn words(text: &str) -> Vec<(usize, usize)> {
     let mut out = Vec::new();
     let mut start: Option<usize> = None;
-    let is_w = |c: char| c.is_alphanumeric() || c == '\'' || c == '’' || c == '-';
+    let is_w = |c: char| c.is_alphanumeric() || c == '\'' || c == '’' || c == '-' || unicode_normalization::char::is_combining_mark(c);
     for (i, c) in text.char_indices() {
         match (start, is_w(c)) {
             (None, true) => start = Some(i),
@@ -698,8 +699,13 @@ fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options
 fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) {
     let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0);
     let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len());
-    let token = text.get(ts..te).unwrap_or("");
-    (ts, te, token.contains("://") || token.contains('@') || token.starts_with("www."))
+    (ts, te, text.get(ts..te).is_some_and(is_address))
+}
+
+/// Does a whitespace-delimited token look like a web or e-mail address (`https://…`, `www.…`,
+/// `a@b.org`)? Spelling skips its words, and Spanish grammar its punctuation.
+pub(crate) fn is_address(token: &str) -> bool {
+    token.contains("://") || token.contains('@') || token.starts_with("www.")
 }
 
 /// The spelling issue of the word at `a..b` of `text`, if it is misspelled in `lang`. Words in
@@ -731,8 +737,21 @@ pub fn is_correct_in(word: &str, lang: ProofLang) -> bool {
     is_correct_in_with(word, lang, &Options::default())
 }
 
-/// [`is_correct_in`] with the File › Options › Proofing choices.
+/// `word` composed (NFC), as the dictionaries spell their words: `tambie\u{301}n` is looked up as
+/// `también`. Words already composed (all ASCII ones among them) are borrowed as they are.
+fn composed(word: &str) -> std::borrow::Cow<'_, str> {
+    use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
+    if word.is_ascii() || is_nfc_quick(word.chars()) == IsNormalized::Yes {
+        std::borrow::Cow::Borrowed(word)
+    } else {
+        std::borrow::Cow::Owned(word.nfc().collect())
+    }
+}
+
+/// [`is_correct_in`] with the File › Options › Proofing choices. A word with combining marks is
+/// checked as its composed form ([`composed`]).
 pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
+    let word = &*composed(word);
     match lang {
         ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts),
         ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct(word),
@@ -750,8 +769,10 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
     }
 }
 
-/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`).
+/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed
+/// ([`composed`]) like the dictionaries' words.
 pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
+    let word = &*composed(word);
     match lang {
         ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang),
         ProofLang::Pl => pl::suggest(word, max),
@@ -1617,6 +1638,20 @@ mod tests {
         }
     }
 
+    /// Combining marks stay in their word (#3562 review): a decomposed word is one word, with the
+    /// byte range of what was typed, and it is checked as its composed form in every language.
+    #[test]
+    fn decomposed_words_stay_whole() {
+        let text = "tambie\u{301}n, cafe\u{301}  \u{301}x";
+        let ws: Vec<&str> = words(text).into_iter().map(|(a, b)| &text[a..b]).collect();
+        assert_eq!(ws, ["tambie\u{301}n", "cafe\u{301}", "\u{301}x"]);
+        assert_eq!(words("tambie\u{301}n"), [(0, 9)]);
+        assert_eq!(&*composed("tambie\u{301}n"), "también");
+        assert!(matches!(composed("también"), std::borrow::Cow::Borrowed(_)) && matches!(composed("plain"), std::borrow::Cow::Borrowed(_)));
+        pt::test_dict::install();
+        assert!(is_correct_in("corac\u{327}a\u{303}o", ProofLang::Pt), "Portuguese gets the same composition");
+    }
+
     /// #100: runs in a language without a dictionary are never flagged; English still is.
     #[test]
     fn non_english_words_are_not_flagged() {