diff --git a/crates/text/src/bidi.rs b/crates/text/src/bidi.rs index 3be4db3e..ac9b3afd 100644 --- a/crates/text/src/bidi.rs +++ b/crates/text/src/bidi.rs @@ -102,7 +102,9 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo } } // W4/W5 in right-to-left runs: a separator between digits, and currency or percent signs next - // to them, belong to the number. + // to them, belong to the number. `after_et[i]` is the first index after `i` that is not ET, + // computed once so a long run of currency signs costs linear work, not a scan per sign. + let after_et = next_non_et(&cls); for i in 0..n { if !rl(i) || dir.get(i) != Some(&Dir::Open) { continue; @@ -111,7 +113,7 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo let num = |j: Option| j.and_then(|j| dir.get(j)).is_some_and(|d| *d == Dir::Num); let joined = match c { ES | CS => num(i.checked_sub(1)) && num(Some(i + 1)), - ET => num(i.checked_sub(1)) || (i + 1..n).find(|&j| cls.get(j) != Some(&ET)).is_some_and(|j| num(Some(j))), + ET => num(i.checked_sub(1)) || num(after_et.get(i).copied().flatten()), _ => false, }; if joined && let Some(x) = dir.get_mut(i) { @@ -135,9 +137,21 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo Some(Dir::R | Dir::Num) => Some(Dir::R), _ => None, }; + // Resolving a span only changes characters before the scan position, so the nearest strong + // direction after a span can be computed once up front, and the one before it is carried along: + // linear work however many neutral spans the paragraph has. + let mut next_strong: Vec> = vec![None; n + 1]; + for j in (0..n).rev() { + let here = strong_at(&dir, j).or(next_strong.get(j + 1).copied().flatten()); + if let Some(x) = next_strong.get_mut(j) { + *x = here; + } + } + let mut last_strong: Option = None; let mut i = 0; while i < n { if !(rl(i) && dir.get(i) == Some(&Dir::Open)) { + last_strong = strong_at(&dir, i).or(last_strong); i += 1; continue; } @@ -145,12 +159,15 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo while i < n && rl(i) && dir.get(i) == Some(&Dir::Open) { i += 1; } - let before = (0..start).rev().find_map(|j| strong_at(&dir, j)).unwrap_or(sos); - let after = (i..n).find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let before = last_strong.unwrap_or(sos); + let after = next_strong.get(i).copied().flatten().unwrap_or(sos); let d = if before == after { before } else { Dir::R }; for x in dir.get_mut(start..i).into_iter().flatten() { *x = d; } + if start < i { + last_strong = Some(if d == Dir::Num { Dir::R } else { d }); + } } // Spaces in left-to-right runs follow the character before them. let mut prev = Dir::L; @@ -170,6 +187,21 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo .collect() } +/// For each index, the first later index whose class is not ET (`None` when only ET follows). +fn next_non_et(cls: &[BidiClass]) -> Vec> { + let mut out = vec![None; cls.len()]; + let mut next = None; + for i in (0..cls.len()).rev() { + if let Some(x) = out.get_mut(i) { + *x = next; + } + if cls.get(i) != Some(&ET) { + next = Some(i); + } + } + out +} + /// Whitespace-like for rule L1: reset to the paragraph level when trailing a line or preceding a /// segment/paragraph separator. fn l1_whitespace(c: BidiClass) -> bool { @@ -326,4 +358,182 @@ mod tests { assert_eq!(visual_order(&[1, 2, 2, 1]), vec![3, 1, 2, 0]); assert_eq!(visual_order(&[125, 125]), vec![1, 0]); } + + /// The pre-#403 `levels_lang`, kept as the reference the linear version must match. + fn reference_levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[bool]) -> Vec { + let base = u8::from(rtl); + let any_rtl_lang = rtl_lang.iter().any(|r| *r); + if !rtl && !any_rtl_lang && !needs_bidi(chars) { + return vec![0; chars.len()]; + } + let classes: Vec = chars.iter().map(|c| bidi_class(*c)).collect(); + if classes.iter().any(|c| matches!(c, LRE | RLE | LRO | RLO | PDF | LRI | RLI | FSI | PDI)) { + return levels(text, chars, rtl); + } + let n = chars.len(); + let rl = |i: usize| rtl_lang.get(i).copied().unwrap_or(false); + let sos = if rtl { Dir::R } else { Dir::L }; + // W1: combining marks take the class of the character they sit on. + let mut cls = classes; + for i in 0..n { + if cls.get(i) == Some(&NSM) { + let prev = if i == 0 { if rtl { R } else { L } } else { cls.get(i - 1).copied().unwrap_or(L) }; + if let Some(c) = cls.get_mut(i) { + *c = prev; + } + } + } + let mut dir = vec![Dir::Open; n]; + // Strong letters and digits. + let mut last_strong: Option = None; + for i in 0..n { + let c = cls.get(i).copied().unwrap_or(ON); + let d = match c { + L => Dir::L, + R | AL => Dir::R, + AN => Dir::Num, + EN if !rl(i) => Dir::L, + // W7: European digits after left-to-right text are left to right. + EN if last_strong == Some(L) => Dir::L, + EN => Dir::Num, + _ => Dir::Open, + }; + if matches!(c, L | R | AL) { + last_strong = Some(c); + } + if let Some(x) = dir.get_mut(i) { + *x = d; + } + } + // W4/W5 in right-to-left runs: a separator between digits, and currency or percent signs next + // to them, belong to the number. + for i in 0..n { + if !rl(i) || dir.get(i) != Some(&Dir::Open) { + continue; + } + let c = cls.get(i).copied().unwrap_or(ON); + let num = |j: Option| j.and_then(|j| dir.get(j)).is_some_and(|d| *d == Dir::Num); + let joined = match c { + ES | CS => num(i.checked_sub(1)) && num(Some(i + 1)), + ET => num(i.checked_sub(1)) || (i + 1..n).find(|&j| cls.get(j) != Some(&ET)).is_some_and(|j| num(Some(j))), + _ => false, + }; + if joined && let Some(x) = dir.get_mut(i) { + *x = Dir::Num; + } + } + // Punctuation in left-to-right runs is left to right. + for i in 0..n { + if !rl(i) + && dir.get(i) == Some(&Dir::Open) + && !matches!(cls.get(i), Some(WS | S)) + && let Some(x) = dir.get_mut(i) + { + *x = Dir::L; + } + } + // Neutrals in right-to-left runs: the direction around them when both sides agree, else + // right to left. Unresolved spaces of left-to-right runs are looked through. + let strong_at = |dir: &[Dir], j: usize| match dir.get(j) { + Some(Dir::L) => Some(Dir::L), + Some(Dir::R | Dir::Num) => Some(Dir::R), + _ => None, + }; + let mut i = 0; + while i < n { + if !(rl(i) && dir.get(i) == Some(&Dir::Open)) { + i += 1; + continue; + } + let start = i; + while i < n && rl(i) && dir.get(i) == Some(&Dir::Open) { + i += 1; + } + let before = (0..start).rev().find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let after = (i..n).find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let d = if before == after { before } else { Dir::R }; + for x in dir.get_mut(start..i).into_iter().flatten() { + *x = d; + } + } + // Spaces in left-to-right runs follow the character before them. + let mut prev = Dir::L; + for d in dir.iter_mut() { + if *d == Dir::Open { + *d = prev; + } + prev = if *d == Dir::Num { Dir::R } else { *d }; + } + dir.iter() + .map(|d| match (d, base % 2 == 1) { + (Dir::L, false) | (Dir::R, true) => base, + (Dir::L, true) | (Dir::R, false) | (Dir::Open, _) => base + 1, + (Dir::Num, false) => base + 2, + (Dir::Num, true) => base + 1, + }) + .collect() + } + + fn lang(s: &str, rtl: bool, rtl_lang: &[bool]) -> Vec { + let chars: Vec = s.chars().collect(); + levels_lang(s, &chars, rtl, rtl_lang) + } + + #[test] + fn next_non_et_points_past_each_et_run() { + let cls: Vec = "a$$1%".chars().map(bidi_class).collect(); + assert_eq!(next_non_et(&cls), vec![Some(3), Some(3), Some(3), None, None]); + } + + #[test] + fn currency_signs_join_numbers_on_either_side_in_rtl_runs() { + // Hebrew word, then "$$5" and "5%%": the signs travel with the digits. + let s = "שלום $$5 5%% $$"; + let n = s.chars().count(); + let l = lang(s, true, &vec![true; n]); + let chars: Vec = s.chars().collect(); + let num = l[chars.iter().position(|c| *c == '5').unwrap()]; + assert_eq!(&l[5..8], &[num; 3], "{l:?}"); + assert_eq!(&l[9..12], &[num; 3], "{l:?}"); + // Signs with no number next to them stay with the right-to-left text. + assert_eq!(l[13] % 2, 1); + assert_ne!(l[13], num); + } + + #[test] + fn long_currency_and_neutral_runs_are_linear() { + // Quadratic resolution took minutes on these; linear finishes at once even unoptimised. + let n = 200_000; + let dollars: String = std::iter::once('ש').chain(std::iter::repeat_n('$', n)).collect(); + let l = lang(&dollars, true, &vec![true; n + 1]); + assert!(l.iter().all(|v| v % 2 == 1)); + // Alternating right-to-left-language neutrals and left-to-right-language spaces: many spans. + let s: String = std::iter::repeat_n("! ", n / 2).collect(); + let langs: Vec = (0..n).map(|i| i % 2 == 0).collect(); + assert_eq!(lang(&s, true, &langs).len(), n); + } + + #[test] + fn matches_the_reference_on_random_mixed_text() { + let alphabet: Vec = "aZש ب$%#+-,.:1٣ ()!\u{300}".chars().collect(); + let mut seed = 0x2545_f491_u32; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + for _ in 0..4000 { + let len = (next() % 24) as usize; + let chars: Vec = (0..len).map(|_| alphabet[next() as usize % alphabet.len()]).collect(); + let text: String = chars.iter().collect(); + let langs: Vec = (0..len).map(|_| next() % 3 != 0).collect(); + let rtl = next() % 2 == 0; + assert_eq!( + levels_lang(&text, &chars, rtl, &langs), + reference_levels_lang(&text, &chars, rtl, &langs), + "{text:?} rtl={rtl} langs={langs:?}" + ); + } + } }