From cb26e7a04fc77bd02f05ea6922e73d74f91e2d5e Mon Sep 17 00:00:00 2001 From: Nikita Rakitin Date: Tue, 13 Oct 2026 20:52:15 +0300 Subject: [PATCH] Text: bidi levels for language runs in linear time (#403) Currency and percent signs in right-to-left runs looked ahead through the whole ET run for every sign, and each neutral span searched both ways for the nearest strong direction. Both are now precomputed or carried along, so a long run of signs or many short neutral spans cost linear work. The old code stays in the tests as a reference and 4000 random paragraphs match it. Co-authored-by: Cursor --- crates/text/src/bidi.rs | 218 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 4 deletions(-) diff --git a/crates/text/src/bidi.rs b/crates/text/src/bidi.rs index 3be4db3e..ac9b3afd 100644 --- a/crates/text/src/bidi.rs +++ b/crates/text/src/bidi.rs @@ -102,7 +102,9 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo } } // W4/W5 in right-to-left runs: a separator between digits, and currency or percent signs next - // to them, belong to the number. + // to them, belong to the number. `after_et[i]` is the first index after `i` that is not ET, + // computed once so a long run of currency signs costs linear work, not a scan per sign. + let after_et = next_non_et(&cls); for i in 0..n { if !rl(i) || dir.get(i) != Some(&Dir::Open) { continue; @@ -111,7 +113,7 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo let num = |j: Option| j.and_then(|j| dir.get(j)).is_some_and(|d| *d == Dir::Num); let joined = match c { ES | CS => num(i.checked_sub(1)) && num(Some(i + 1)), - ET => num(i.checked_sub(1)) || (i + 1..n).find(|&j| cls.get(j) != Some(&ET)).is_some_and(|j| num(Some(j))), + ET => num(i.checked_sub(1)) || num(after_et.get(i).copied().flatten()), _ => false, }; if joined && let Some(x) = dir.get_mut(i) { @@ -135,9 +137,21 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo Some(Dir::R | Dir::Num) => Some(Dir::R), _ => None, }; + // Resolving a span only changes characters before the scan position, so the nearest strong + // direction after a span can be computed once up front, and the one before it is carried along: + // linear work however many neutral spans the paragraph has. + let mut next_strong: Vec> = vec![None; n + 1]; + for j in (0..n).rev() { + let here = strong_at(&dir, j).or(next_strong.get(j + 1).copied().flatten()); + if let Some(x) = next_strong.get_mut(j) { + *x = here; + } + } + let mut last_strong: Option = None; let mut i = 0; while i < n { if !(rl(i) && dir.get(i) == Some(&Dir::Open)) { + last_strong = strong_at(&dir, i).or(last_strong); i += 1; continue; } @@ -145,12 +159,15 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo while i < n && rl(i) && dir.get(i) == Some(&Dir::Open) { i += 1; } - let before = (0..start).rev().find_map(|j| strong_at(&dir, j)).unwrap_or(sos); - let after = (i..n).find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let before = last_strong.unwrap_or(sos); + let after = next_strong.get(i).copied().flatten().unwrap_or(sos); let d = if before == after { before } else { Dir::R }; for x in dir.get_mut(start..i).into_iter().flatten() { *x = d; } + if start < i { + last_strong = Some(if d == Dir::Num { Dir::R } else { d }); + } } // Spaces in left-to-right runs follow the character before them. let mut prev = Dir::L; @@ -170,6 +187,21 @@ pub(crate) fn levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[boo .collect() } +/// For each index, the first later index whose class is not ET (`None` when only ET follows). +fn next_non_et(cls: &[BidiClass]) -> Vec> { + let mut out = vec![None; cls.len()]; + let mut next = None; + for i in (0..cls.len()).rev() { + if let Some(x) = out.get_mut(i) { + *x = next; + } + if cls.get(i) != Some(&ET) { + next = Some(i); + } + } + out +} + /// Whitespace-like for rule L1: reset to the paragraph level when trailing a line or preceding a /// segment/paragraph separator. fn l1_whitespace(c: BidiClass) -> bool { @@ -326,4 +358,182 @@ mod tests { assert_eq!(visual_order(&[1, 2, 2, 1]), vec![3, 1, 2, 0]); assert_eq!(visual_order(&[125, 125]), vec![1, 0]); } + + /// The pre-#403 `levels_lang`, kept as the reference the linear version must match. + fn reference_levels_lang(text: &str, chars: &[char], rtl: bool, rtl_lang: &[bool]) -> Vec { + let base = u8::from(rtl); + let any_rtl_lang = rtl_lang.iter().any(|r| *r); + if !rtl && !any_rtl_lang && !needs_bidi(chars) { + return vec![0; chars.len()]; + } + let classes: Vec = chars.iter().map(|c| bidi_class(*c)).collect(); + if classes.iter().any(|c| matches!(c, LRE | RLE | LRO | RLO | PDF | LRI | RLI | FSI | PDI)) { + return levels(text, chars, rtl); + } + let n = chars.len(); + let rl = |i: usize| rtl_lang.get(i).copied().unwrap_or(false); + let sos = if rtl { Dir::R } else { Dir::L }; + // W1: combining marks take the class of the character they sit on. + let mut cls = classes; + for i in 0..n { + if cls.get(i) == Some(&NSM) { + let prev = if i == 0 { if rtl { R } else { L } } else { cls.get(i - 1).copied().unwrap_or(L) }; + if let Some(c) = cls.get_mut(i) { + *c = prev; + } + } + } + let mut dir = vec![Dir::Open; n]; + // Strong letters and digits. + let mut last_strong: Option = None; + for i in 0..n { + let c = cls.get(i).copied().unwrap_or(ON); + let d = match c { + L => Dir::L, + R | AL => Dir::R, + AN => Dir::Num, + EN if !rl(i) => Dir::L, + // W7: European digits after left-to-right text are left to right. + EN if last_strong == Some(L) => Dir::L, + EN => Dir::Num, + _ => Dir::Open, + }; + if matches!(c, L | R | AL) { + last_strong = Some(c); + } + if let Some(x) = dir.get_mut(i) { + *x = d; + } + } + // W4/W5 in right-to-left runs: a separator between digits, and currency or percent signs next + // to them, belong to the number. + for i in 0..n { + if !rl(i) || dir.get(i) != Some(&Dir::Open) { + continue; + } + let c = cls.get(i).copied().unwrap_or(ON); + let num = |j: Option| j.and_then(|j| dir.get(j)).is_some_and(|d| *d == Dir::Num); + let joined = match c { + ES | CS => num(i.checked_sub(1)) && num(Some(i + 1)), + ET => num(i.checked_sub(1)) || (i + 1..n).find(|&j| cls.get(j) != Some(&ET)).is_some_and(|j| num(Some(j))), + _ => false, + }; + if joined && let Some(x) = dir.get_mut(i) { + *x = Dir::Num; + } + } + // Punctuation in left-to-right runs is left to right. + for i in 0..n { + if !rl(i) + && dir.get(i) == Some(&Dir::Open) + && !matches!(cls.get(i), Some(WS | S)) + && let Some(x) = dir.get_mut(i) + { + *x = Dir::L; + } + } + // Neutrals in right-to-left runs: the direction around them when both sides agree, else + // right to left. Unresolved spaces of left-to-right runs are looked through. + let strong_at = |dir: &[Dir], j: usize| match dir.get(j) { + Some(Dir::L) => Some(Dir::L), + Some(Dir::R | Dir::Num) => Some(Dir::R), + _ => None, + }; + let mut i = 0; + while i < n { + if !(rl(i) && dir.get(i) == Some(&Dir::Open)) { + i += 1; + continue; + } + let start = i; + while i < n && rl(i) && dir.get(i) == Some(&Dir::Open) { + i += 1; + } + let before = (0..start).rev().find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let after = (i..n).find_map(|j| strong_at(&dir, j)).unwrap_or(sos); + let d = if before == after { before } else { Dir::R }; + for x in dir.get_mut(start..i).into_iter().flatten() { + *x = d; + } + } + // Spaces in left-to-right runs follow the character before them. + let mut prev = Dir::L; + for d in dir.iter_mut() { + if *d == Dir::Open { + *d = prev; + } + prev = if *d == Dir::Num { Dir::R } else { *d }; + } + dir.iter() + .map(|d| match (d, base % 2 == 1) { + (Dir::L, false) | (Dir::R, true) => base, + (Dir::L, true) | (Dir::R, false) | (Dir::Open, _) => base + 1, + (Dir::Num, false) => base + 2, + (Dir::Num, true) => base + 1, + }) + .collect() + } + + fn lang(s: &str, rtl: bool, rtl_lang: &[bool]) -> Vec { + let chars: Vec = s.chars().collect(); + levels_lang(s, &chars, rtl, rtl_lang) + } + + #[test] + fn next_non_et_points_past_each_et_run() { + let cls: Vec = "a$$1%".chars().map(bidi_class).collect(); + assert_eq!(next_non_et(&cls), vec![Some(3), Some(3), Some(3), None, None]); + } + + #[test] + fn currency_signs_join_numbers_on_either_side_in_rtl_runs() { + // Hebrew word, then "$$5" and "5%%": the signs travel with the digits. + let s = "שלום $$5 5%% $$"; + let n = s.chars().count(); + let l = lang(s, true, &vec![true; n]); + let chars: Vec = s.chars().collect(); + let num = l[chars.iter().position(|c| *c == '5').unwrap()]; + assert_eq!(&l[5..8], &[num; 3], "{l:?}"); + assert_eq!(&l[9..12], &[num; 3], "{l:?}"); + // Signs with no number next to them stay with the right-to-left text. + assert_eq!(l[13] % 2, 1); + assert_ne!(l[13], num); + } + + #[test] + fn long_currency_and_neutral_runs_are_linear() { + // Quadratic resolution took minutes on these; linear finishes at once even unoptimised. + let n = 200_000; + let dollars: String = std::iter::once('ש').chain(std::iter::repeat_n('$', n)).collect(); + let l = lang(&dollars, true, &vec![true; n + 1]); + assert!(l.iter().all(|v| v % 2 == 1)); + // Alternating right-to-left-language neutrals and left-to-right-language spaces: many spans. + let s: String = std::iter::repeat_n("! ", n / 2).collect(); + let langs: Vec = (0..n).map(|i| i % 2 == 0).collect(); + assert_eq!(lang(&s, true, &langs).len(), n); + } + + #[test] + fn matches_the_reference_on_random_mixed_text() { + let alphabet: Vec = "aZש ب$%#+-,.:1٣ ()!\u{300}".chars().collect(); + let mut seed = 0x2545_f491_u32; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + for _ in 0..4000 { + let len = (next() % 24) as usize; + let chars: Vec = (0..len).map(|_| alphabet[next() as usize % alphabet.len()]).collect(); + let text: String = chars.iter().collect(); + let langs: Vec = (0..len).map(|_| next() % 3 != 0).collect(); + let rtl = next() % 2 == 0; + assert_eq!( + levels_lang(&text, &chars, rtl, &langs), + reference_levels_lang(&text, &chars, rtl, &langs), + "{text:?} rtl={rtl} langs={langs:?}" + ); + } + } }