diff --git a/crates/layout/src/para.rs b/crates/layout/src/para.rs index e23b61749..6b1ea46fd 100644 --- a/crates/layout/src/para.rs +++ b/crates/layout/src/para.rs @@ -2422,7 +2422,9 @@ fn hyphen_glyph(pl: &ParaLayout, style: u32, cache: &mut Vec<(u32, u32, f32)>) - /// paragraph's text. fn hyphenation_points(text: &str, to_para: impl Fn(usize) -> Option, pl: &ParaLayout, auto: bool, caps: bool, runs: &[ProofRun]) -> Vec { let mut bytes: Vec = text.char_indices().filter(|(_, c)| *c == SOFT_HYPHEN).map(|(i, c)| i + c.len_utf8()).collect(); - if auto { + // Vietnamese words are single syllables and never take a hyphen: no automatic hyphenation in + // a Vietnamese paragraph, nor in a word written with a Vietnamese-only letter (`Nguyễn`). + if auto && !wordcraft_proof::vi::looks_vietnamese(text) { let lim = wordcraft_proof::hyphen::Limits::default(); let mut start: Option = None; for (i, c) in text.char_indices().chain(std::iter::once((text.len(), ' '))) { @@ -2436,6 +2438,7 @@ fn hyphenation_points(text: &str, to_para: impl Fn(usize) -> Option, pl: if let Some(w) = text.get(a..i) && w.chars().count() >= lim.min_word && !w.chars().any(wordcraft_fonts::is_rtl) + && !wordcraft_proof::vi::is_vietnamese_word(w) && (caps || !all_caps(w)) { // The word's language: its first letter's run, found by its place in `p.text`. diff --git a/crates/layout/src/tests.rs b/crates/layout/src/tests.rs index 1c8542b4b..ef4c182b5 100644 --- a/crates/layout/src/tests.rs +++ b/crates/layout/src/tests.rs @@ -6388,6 +6388,23 @@ fn automatic_spacing_drops_between_list_items() { } } +/// Vietnamese is written in single syllables: automatic hyphenation leaves its words whole. (The +/// English patterns happen not to split these accented words today; the Vietnamese guard in +/// `hyphenation_points` keeps it so whatever the patterns become.) +#[test] +fn automatic_hyphenation_leaves_vietnamese_whole() { + let text = + "Những người khuyến khích chuyện trường thường nghiêng nghiêng nhưng thành phố Nguyễn Trường Thượng xuyên suốt chương trình. ".repeat(8); + let mut d = Document::from_text(&text); + d.settings.auto_hyphenation = true; + let hyphens: usize = lay(&d).pages[0] + .items + .iter() + .filter_map(|i| if let Placed::Lines { para, .. } = i { Some(para.lines.iter().filter(|x| x.hyphen.is_some()).count()) } else { None }) + .sum(); + assert_eq!(hyphens, 0); +} + /// #1252: highlight and character-shading fills are marked as text backgrounds (a dark page dims /// them), and so is the text drawn on them; a paragraph's shading is an ordinary fill. #[test] diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 369025527..a0099c664 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -49,6 +49,7 @@ pub mod pl; pub mod pt; pub mod ru; pub mod tr; +pub mod vi; use std::collections::HashSet; use std::ops::Range; @@ -581,7 +582,8 @@ fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { /// classification of the word's enclosing token, from [`enclosing_token`]. fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Options, internet: bool) -> Option { let w = text.get(a..b)?; - if (opts.ignore_internet && internet) || is_correct_in_with(w, lang, opts) { + // A word with a Vietnamese-only letter (a name like Nguyễn in English text) isn't English. + if (opts.ignore_internet && internet) || (lang == ProofLang::En && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { return None; } let message = match lang { @@ -668,9 +670,19 @@ pub fn check_text_with(text: &str, spans: &[(Range, ProofLang)], opts: &O _ => merged.push((r.clone(), *l)), } } + // A paragraph typed in Vietnamese under the default English language is Vietnamese, which + // has no proofing tools: no marks (Word detects the language the same way). + let default = if vi::looks_vietnamese(text) { + for (_, l) in merged.iter_mut().filter(|(_, l)| *l == ProofLang::En) { + *l = ProofLang::Other; + } + ProofLang::Other + } else { + ProofLang::En + }; let lang_at = |i: usize| -> ProofLang { let k = merged.partition_point(|(r, _)| r.end <= i); - merged.get(k).filter(|(r, _)| r.start <= i).map_or(ProofLang::En, |s| s.1) + merged.get(k).filter(|(r, _)| r.start <= i).map_or(default, |s| s.1) }; let mut v: Vec = spelling_issues(text, lang_at, opts); if !grammar { @@ -686,7 +698,7 @@ pub fn check_text_with(text: &str, spans: &[(Range, ProofLang)], opts: &O covered = covered.max(r.end); } if covered < text.len() || merged.is_empty() { - langs.push(ProofLang::En); + langs.push(default); } langs.sort(); langs.dedup(); diff --git a/crates/proof/src/vi.rs b/crates/proof/src/vi.rs new file mode 100644 index 000000000..bb24b88e9 --- /dev/null +++ b/crates/proof/src/vi.rs @@ -0,0 +1,93 @@ +//! Telling Vietnamese text apart from English, so a document typed in Vietnamese under the default +//! English proofing language isn't covered in red underlines (as Word's "detect language +//! automatically" does). There are no Vietnamese proofing tools yet, so Vietnamese text is treated +//! like any language without them: no spelling or grammar marks. +//! +//! The test is orthographic. Vietnamese writes most syllables with letters no other Latin-script +//! language shares: the horn letters (`ơ ư`), `ĩ ũ`, and the tone-marked vowels of the Latin +//! Extended Additional block (`ạ ả ấ ầ … ỹ`), precomposed or as combining hook above, dot below +//! and horn. French and Portuguese share `â ê ô à é`, Romanian `ă` and Croatian, Serbian and +//! Bosnian `đ`, so those alone don't count. + +/// Does `c` occur in Vietnamese words but (practically) nowhere else? +fn vietnamese_letter(c: char) -> bool { + matches!(c, 'ơ' | 'Ơ' | 'ư' | 'Ư' | 'ĩ' | 'Ĩ' | 'ũ' | 'Ũ' | '\u{1EA0}'..='\u{1EF9}' | '\u{0309}' | '\u{0323}' | '\u{031B}') +} + +/// Is `word` written with a Vietnamese-only letter (`Việt`, `ơn`)? Such a word is never an +/// English misspelling. +pub fn is_vietnamese_word(word: &str) -> bool { + let combining = |c: char| matches!(c, '\u{0300}'..='\u{036F}'); + // Two marks stacked on one letter (decomposed `ế` = e + circumflex + acute) are Vietnamese too. + word.chars().any(vietnamese_letter) || word.chars().zip(word.chars().skip(1)).any(|(a, b)| combining(a) && combining(b)) +} + +/// Does a paragraph read as Vietnamese? At least two of its words (any, in a heading of up to three +/// words) carry a Vietnamese-only letter, and they are at least a sixth of its words: Vietnamese syllables without +/// such a letter (`anh`, `con`, `ba`) are common, while an English paragraph quoting one +/// Vietnamese name stays English. +pub fn looks_vietnamese(text: &str) -> bool { + let mut total = 0usize; + let mut marked = 0usize; + for w in text.split(|c: char| !(c.is_alphanumeric() || matches!(c, '\u{0300}'..='\u{036F}'))).filter(|w| !w.is_empty()) { + total += 1; + if is_vietnamese_word(w) { + marked += 1; + } + } + marked > 0 && (marked >= 2 || total <= 3) && marked * 6 >= total +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vietnamese_words() { + for w in ["Việt", "ơn", "những", "Ước", "mũi", "Nghĩa", "tie\u{0302}\u{0301}ng", "nho\u{031B}"] { + assert!(is_vietnamese_word(w), "{w}"); + } + // Shared with French, Portuguese, Spanish: not enough on their own. + // Romanian ă and Croatian đ are Vietnamese letters too, but not Vietnamese-only. + for w in ["café", "être", "côte", "água", "señor", "anh", "the", "mănânc", "acasă", "Đurđa", "đak", "đi"] { + assert!(!is_vietnamese_word(w), "{w}"); + } + } + + #[test] + fn paragraphs() { + assert!(looks_vietnamese("Tôi yêu tiếng Việt và những bài thơ của nó.")); + assert!(looks_vietnamese("Hà Nội")); + assert!(looks_vietnamese("Việt")); + // English quoting one Vietnamese name, and French, stay as they are. + assert!(!looks_vietnamese("The meeting with Mr Nguyễn is on Tuesday at the main office.")); + assert!(!looks_vietnamese("Le café de la côte est très bon.")); + assert!(!looks_vietnamese("")); + assert!(!looks_vietnamese("anh em")); + // Romanian and Croatian paragraphs, full of ă and đ, aren't Vietnamese. + assert!(!looks_vietnamese("Mă duc acasă să mănânc cu fratele meu, că e târziu.")); + assert!(!looks_vietnamese("Đurđa i Đorđe su se vratili kući, a Đakovo je daleko.")); + } +} + +#[cfg(test)] +mod checker_tests { + use crate::{Options, ProofLang, check_text_with}; + + #[test] + fn vietnamese_paragraphs_under_english_get_no_marks() { + let text = "Hôm nay tôi đi học, trời rất đẹp và mát mẻ."; + let spans = [(0..text.len(), ProofLang::En)]; + assert!(check_text_with(text, &spans, &Options::default(), true).is_empty()); + // No language at all (the document default) behaves the same. + assert!(check_text_with(text, &[], &Options::default(), true).is_empty()); + } + + #[test] + fn english_keeps_its_marks_and_skips_vietnamese_names() { + let text = "We met Nguyễn yesterdy at the office."; + let issues = check_text_with(text, &[(0..text.len(), ProofLang::En)], &Options::default(), false); + let flagged: Vec<&str> = issues.iter().filter_map(|i| text.get(i.start..i.end)).collect(); + assert_eq!(flagged, ["yesterdy"]); + } +}