From 441e80b7a66ffb6cd9a0516564fc4ebd7fd14a6e Mon Sep 17 00:00:00 2001 From: Earl Co Date: Fri, 23 Oct 2026 16:52:40 +0200 Subject: [PATCH] security(tts): make malformed-token scanning linear (#2871) * security(tts): make malformed-token scanning linear * tts-text: find a pause marker's ] by byte, so a multi-byte character at byte 64 doesn't hide it Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Brandon Thomas Co-authored-by: Claude Opus 5.5 --- crates/tts-text/src/normalize.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/tts-text/src/normalize.rs b/crates/tts-text/src/normalize.rs index 46f74075c..34a213c32 100644 --- a/crates/tts-text/src/normalize.rs +++ b/crates/tts-text/src/normalize.rs @@ -44,6 +44,8 @@ const MAX_INPUT: usize = 1_048_576; const MAX_PAUSE_MS: u32 = 10_000; /// Numbers with more than this many digits are read digit by digit (never overflow). const MAX_CARDINAL_DIGITS: usize = 12; +/// A malformed pause marker is only inspected within this bounded prefix. +const MAX_PAUSE_MARKER_BYTES: usize = 64; /// Normalize `text` into spoken-word tokens for `lang`. /// @@ -157,7 +159,9 @@ pub fn normalize(text: &str, lang: Lang) -> Result, NormalizeError> { out.push(terminator); } } - _ if is_word_char(c) => match parse_wordish(rest, lang) { + // Apostrophes can occur inside a word, but an apostrophe-only run must not repeatedly + // scan the whole remaining script looking for a word. + _ if c.is_alphabetic() => match parse_wordish(rest, lang) { Some((tokens, used)) => { pos += used; out.extend(tokens); @@ -202,7 +206,8 @@ fn peek(s: &str, i: usize) -> Option { /// `[pause 1s]`, `[pause 500ms]`, `[pause 1.5s]` → a capped pause. Malformed markers /// return `None` and fall through to plain text (never an error). fn parse_pause(rest: &str) -> Option<(u32, usize)> { - let close = rest.find(']')?; + // search bytes: the 64-byte cut may fall inside a multi-byte character that follows the marker + let close = rest.as_bytes().iter().take(MAX_PAUSE_MARKER_BYTES).position(|&b| b == b']')?; let inner = rest.get(1..close)?.trim(); let value = inner.strip_prefix("pause")?; let value = value.trim_start(); @@ -952,12 +957,13 @@ fn year_words(year: u64, _lang: Lang) -> Vec { /// Words, abbreviations, months, acronyms, a.m./p.m. and e.g./i.e. fn parse_wordish(rest: &str, _lang: Lang) -> Option<(Vec, usize)> { - let collected: String = rest.chars().take_while(|c| is_word_char(*c)).collect(); + let end = rest.char_indices().find_map(|(i, c)| (!is_word_char(c)).then_some(i)).unwrap_or(rest.len()); + let collected = rest.get(..end)?; let word = collected.trim_matches(|c| c == '\'' || c == '’'); if word.is_empty() { return None; } - let used = word.len() + leading_apostrophes(rest); + let used = word.len(); let lower = word.to_ascii_lowercase(); // a.m. / p.m. — a single letter, a dot, "m", and a dot or boundary. @@ -999,10 +1005,6 @@ fn parse_wordish(rest: &str, _lang: Lang) -> Option<(Vec, usize)> { Some((vec![Token::Word(word.into())], used)) } -fn leading_apostrophes(rest: &str) -> usize { - rest.chars().take_while(|c| *c == '\'' || *c == '’').map(char::len_utf8).sum() -} - /// Expansions for `Mr.` etc.; the dot is consumed so it never ends a sentence. /// `St.` picks "saint" before a capitalised name and "street" otherwise. fn dotted_abbreviation(lower: &str, rest: &str, used: usize) -> Option<(Vec, usize)> { @@ -1493,6 +1495,9 @@ mod tests { assert_eq!(words("[pause 0.25s]", Lang::EnUs), "[pause 250]"); assert_eq!(words("[pause 11s]", Lang::EnUs), "[pause 10000]"); assert_eq!(words("[pause 999999999999s]", Lang::EnUs), "[pause 10000]"); + // byte 64 falls inside an `é` after the marker: still a pause + let tokens = normalize(&format!("[pause 1s]x{}", "é".repeat(40)), Lang::EnUs).unwrap(); + assert_eq!(tokens.first(), Some(&Token::Pause { millis: 1000 })); // Malformed markers stay plain text. assert_eq!(words("[pause", Lang::EnUs), "[ pause"); assert_eq!(words("[pause x]", Lang::EnUs), "[ pause x ]"); @@ -1545,6 +1550,14 @@ mod tests { assert_eq!(words("1,2345", Lang::EnUs), "one , two thousand three hundred forty-five"); } + #[test] + fn malformed_marker_and_apostrophe_runs_make_linear_progress() { + let brackets = "[".repeat(20_000); + let apostrophes = "'".repeat(20_000); + assert_eq!(normalize(&brackets, Lang::EnUs).unwrap().len(), 20_000); + assert_eq!(normalize(&apostrophes, Lang::EnUs).unwrap().len(), 20_000); + } + #[test] fn hostile_inputs_do_not_panic() { let hostile: Vec = vec![