diff --git a/crates/ui-egui/src/i18n.rs b/crates/ui-egui/src/i18n.rs index a7df011c2..c9e0e6aea 100644 --- a/crates/ui-egui/src/i18n.rs +++ b/crates/ui-egui/src/i18n.rs @@ -569,6 +569,9 @@ pub fn rules_label(rules: &lightcraft_catalog::RuleSet, catalog: &lightcraft_cat if language() == Locale::En { rules.describe_with(catalog) } else { describe(rules, catalog, 0) } } +#[cfg(test)] +mod gaps; + #[cfg(test)] mod tests; diff --git a/crates/ui-egui/src/i18n/gaps.rs b/crates/ui-egui/src/i18n/gaps.rs new file mode 100644 index 000000000..1cec9746e --- /dev/null +++ b/crates/ui-egui/src/i18n/gaps.rs @@ -0,0 +1,475 @@ +//! The plain `tr("…")` messages of the app's source, for `i18n::tests::translation_gaps_report`. +//! +//! A catalog only holds the strings someone translated, so a string no language has yet is in no +//! catalog and the report could not name it. This reads the source for them: every `tr("…")` / +//! `.tr("…")` call in `crates/ui-egui/src` and `apps/lightcraft/src` whose argument is one string +//! literal (a trailing comma, as rustfmt leaves in a wrapped call, is fine), or an `if`/`match` +//! whose branches are literals (`tr(if on { "On" } else { "Off" })`). Test code (`tests.rs` and +//! `tests_*.rs` files, `tests/` folders and items under exactly `#[cfg(test)]`) is skipped. The scanner lexes +//! the source (comments, strings, raw strings and chars) rather than grepping it, so a `tr(` in a +//! comment or a string is not a use. +//! +//! A key that reaches `tr` through a variable (`tr(label)` over a table of labels, +//! `tr(&format!(…))`) cannot be read from the source; the report takes those labels from its own +//! tables. `tr_ctx` keys (`"|"`) are optional by design. + +use std::{ + collections::BTreeSet, + fs, + path::{Path, PathBuf}, +}; + +/// Source folders whose `tr` calls are read, relative to the crate. +const SOURCES: [&str; 2] = ["src", "../../apps/lightcraft/src"]; + +#[derive(Debug, Clone, PartialEq)] +enum Kind { + Ident(String), + Str(String), + Punct(char), + Other, +} + +#[derive(Debug, Clone)] +struct Tok { + kind: Kind, + line: usize, +} + +/// One `tr` call with a literal key. +#[derive(Debug, Clone, PartialEq)] +struct Use { + key: String, + line: usize, +} + +/// Split Rust source into the tokens the scan needs. Comments vanish; strings, raw strings and +/// chars are single tokens, so braces and parentheses inside them never count. +fn lex(source: &str) -> Vec { + let chars: Vec = source.replace("\r\n", "\n").chars().collect(); + let mut tokens = Vec::new(); + let (mut i, mut line) = (0, 1); + while i < chars.len() { + let c = chars[i]; + let next = chars.get(i + 1).copied(); + if c == '\n' { + line += 1; + i += 1; + } else if c.is_whitespace() { + i += 1; + } else if c == '/' && next == Some('/') { + while i < chars.len() && chars[i] != '\n' { + i += 1; + } + } else if c == '/' && next == Some('*') { + let mut depth = 0; + while i < chars.len() { + if chars[i] == '/' && chars.get(i + 1) == Some(&'*') { + depth += 1; + i += 2; + } else if chars[i] == '*' && chars.get(i + 1) == Some(&'/') { + depth -= 1; + i += 2; + if depth == 0 { + break; + } + } else { + if chars[i] == '\n' { + line += 1; + } + i += 1; + } + } + } else if c == '"' { + let start = line; + let (text, end, lines) = read_string(&chars, i + 1); + tokens.push(Tok { kind: Kind::Str(text), line: start }); + line += lines; + i = end; + } else if c == '\'' { + // A char literal (`'x'`, `'\n'`, `'\u{1F600}'`) is one token; anything else is a lifetime. + let is_char = next == Some('\\') || chars.get(i + 2) == Some(&'\''); + if is_char { + i += if next == Some('\\') { 3 } else { 1 }; + while i < chars.len() && chars[i] != '\'' { + i += 1; + } + i += 1; + tokens.push(Tok { kind: Kind::Other, line }); + } else { + tokens.push(Tok { kind: Kind::Punct('\''), line }); + i += 1; + } + } else if c.is_alphabetic() || c == '_' { + let start = i; + while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { + i += 1; + } + let word: String = chars[start..i].iter().collect(); + let raw_prefix = word == "r" || word == "br"; + let hashes = chars[i..].iter().take_while(|c| **c == '#').count(); + if raw_prefix && chars.get(i + hashes) == Some(&'"') { + let begin = i + hashes + 1; + let closing: Vec = std::iter::once('"').chain(std::iter::repeat_n('#', hashes)).collect(); + let end = (begin..chars.len()).find(|at| chars[*at..].starts_with(&closing)).unwrap_or(chars.len()); + let text: String = chars[begin..end].iter().collect(); + let lines = text.matches('\n').count(); + tokens.push(Tok { kind: Kind::Str(text), line }); + line += lines; + i = (end + closing.len()).min(chars.len()); + } else { + tokens.push(Tok { kind: Kind::Ident(word), line }); + } + } else if c.is_ascii_digit() { + while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { + i += 1; + } + tokens.push(Tok { kind: Kind::Other, line }); + } else { + tokens.push(Tok { kind: Kind::Punct(c), line }); + i += 1; + } + } + tokens +} + +/// Read a (non-raw) string body starting after its opening quote: the decoded text, the index after +/// the closing quote and how many newlines the literal spans. +fn read_string(chars: &[char], mut i: usize) -> (String, usize, usize) { + let (mut text, mut lines) = (String::new(), 0); + while i < chars.len() && chars[i] != '"' { + if chars[i] != '\\' { + if chars[i] == '\n' { + lines += 1; + } + text.push(chars[i]); + i += 1; + continue; + } + i += 1; + match chars.get(i).copied() { + Some('n') => text.push('\n'), + Some('t') => text.push('\t'), + Some('r') => text.push('\r'), + Some('0') => text.push('\0'), + Some('x') => { + let hex: String = chars[i + 1..(i + 3).min(chars.len())].iter().collect(); + text.extend(u8::from_str_radix(&hex, 16).ok().map(char::from)); + i += 2; + } + Some('u') => { + let close = (i..chars.len()).find(|at| chars[*at] == '}').unwrap_or(chars.len() - 1); + let hex: String = chars[i + 2..close].iter().filter(|c| **c != '_').collect(); + text.extend(u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32)); + i = close; + } + // A backslash before a newline joins the lines and drops the next line's indentation. + Some('\n') => { + lines += 1; + while chars.get(i + 1).is_some_and(|c| c.is_whitespace()) { + if chars[i + 1] == '\n' { + lines += 1; + } + i += 1; + } + } + Some(other) => text.push(other), + None => {} + } + i += 1; + } + (text, i + 1, lines) +} + +/// The index of the token that closes the group opened at `open`. +fn matching(tokens: &[Tok], open: usize) -> usize { + let Kind::Punct(opener) = tokens[open].kind else { return open }; + let closer = match opener { + '(' => ')', + '{' => '}', + _ => ']', + }; + let mut depth = 0; + for (at, token) in tokens.iter().enumerate().skip(open) { + match token.kind { + Kind::Punct(c) if c == opener => depth += 1, + Kind::Punct(c) if c == closer => { + depth -= 1; + if depth == 0 { + return at; + } + } + _ => {} + } + } + tokens.len() - 1 +} + +fn is_punct(token: Option<&Tok>, c: char) -> bool { + token.is_some_and(|token| token.kind == Kind::Punct(c)) +} + +fn is_ident(token: Option<&Tok>, name: &str) -> bool { + token.is_some_and(|token| matches!(&token.kind, Kind::Ident(word) if word == name)) +} + +/// Is `tokens[at..]` the attribute `#[cfg(test)]`? +fn is_cfg_test(tokens: &[Tok], at: usize) -> bool { + is_punct(tokens.get(at), '#') + && is_punct(tokens.get(at + 1), '[') + && is_ident(tokens.get(at + 2), "cfg") + && is_punct(tokens.get(at + 3), '(') + && is_ident(tokens.get(at + 4), "test") + && is_punct(tokens.get(at + 5), ')') + && is_punct(tokens.get(at + 6), ']') +} + +/// Does a comma-terminated entry start at `at`: a struct field (or a field in a struct literal, +/// `name: …`) or an enum variant (`Name,` `Name(…)` `Name { … }` `Name = 3`)? +fn starts_entry(tokens: &[Tok], mut at: usize) -> bool { + if is_ident(tokens.get(at), "pub") { + at += 1; + if is_punct(tokens.get(at), '(') { + at = matching(tokens, at) + 1; + } + } + const STATEMENTS: [&str; 10] = ["if", "let", "match", "for", "while", "loop", "unsafe", "return", "else", "break"]; + let Some(Kind::Ident(name)) = tokens.get(at).map(|token| &token.kind) else { return false }; + if STATEMENTS.contains(&name.as_str()) { + return false; + } + let field = is_punct(tokens.get(at + 1), ':') && !is_punct(tokens.get(at + 2), ':'); + let variant = matches!(tokens.get(at + 1).map(|token| &token.kind), Some(Kind::Punct(',' | '(' | '{' | '=' | '}'))); + field || variant +} + +/// The index just past the item that starts at `at` (after its attributes): a struct field, a +/// field in a struct literal or an enum variant ends at its comma, anything else at its closing +/// brace or its `;`. Groups are jumped over, so a `;` in `[u8; 4]` or a `,` in `f(a, b)` ends nothing. +fn skip_item(tokens: &[Tok], mut at: usize) -> usize { + while is_punct(tokens.get(at), '#') && is_punct(tokens.get(at + 1), '[') { + at = matching(tokens, at + 1) + 1; + } + let entry = starts_entry(tokens, at); + while let Some(token) = tokens.get(at) { + match token.kind { + Kind::Punct('(' | '[') => at = matching(tokens, at) + 1, + Kind::Punct('{') if entry => at = matching(tokens, at) + 1, + Kind::Punct('{') => return matching(tokens, at) + 1, + Kind::Punct(';') => return at + 1, + Kind::Punct(',') if entry => return at + 1, + // The enclosing block ends: the last entry has no comma. + Kind::Punct('}' | ')' | ']') if entry => return at, + _ => at += 1, + } + } + at +} + +/// The literal keys of every non-test `tr(…)` call in one source file. +fn scan_source(source: &str) -> Vec { + let tokens = lex(source); + let mut found = Vec::new(); + let mut at = 0; + while at < tokens.len() { + if is_cfg_test(&tokens, at) { + at = skip_item(&tokens, at + 7); + continue; + } + let is_call = is_ident(tokens.get(at), "tr") && is_punct(tokens.get(at + 1), '(') && !(at > 0 && is_ident(tokens.get(at - 1), "fn")); + if is_call { + let close = matching(&tokens, at + 1); + let mut inner = &tokens[at + 2..close]; + // rustfmt wraps a long call as `tr(\n "…",\n)`: the trailing comma is not an argument. + if let [rest @ .., Tok { kind: Kind::Punct(','), .. }] = inner { + inner = rest; + } + let mut keys = Vec::new(); + match inner { + [Tok { kind: Kind::Str(key), line }] => keys.push((key.clone(), *line)), + // `tr(if flag { "On" } else { "Off" })` and `tr(match x { A => "a", B => { "b" } })`: + // each literal that is a whole branch is a key. + [Tok { kind: Kind::Ident(word), .. }, ..] if word == "if" || word == "match" => { + for window in inner.windows(3) { + if let [before, Tok { kind: Kind::Str(key), line }, after] = window { + let braced = before.kind == Kind::Punct('{') && after.kind == Kind::Punct('}'); + let arm = before.kind == Kind::Punct('>') && matches!(after.kind, Kind::Punct(',' | '}')); + if braced || arm { + keys.push((key.clone(), *line)); + } + } + } + } + _ => {} + } + found.extend(keys.into_iter().filter(|(key, _)| !key.is_empty()).map(|(key, line)| Use { key, line })); + } + at += 1; + } + found +} + +/// Every `.rs` file under `dir` that is not a test file, in a stable order. +fn source_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { return }; + let mut paths: Vec = entries.filter_map(|entry| entry.ok().map(|entry| entry.path())).collect(); + paths.sort(); + for path in paths { + let name = path.file_name().and_then(|name| name.to_str()).unwrap_or_default(); + if path.is_dir() { + if name != "tests" { + source_files(&path, out); + } + } else if name.ends_with(".rs") && name != "tests.rs" && !name.starts_with("tests_") { + out.push(path); + } + } +} + +/// The literal keys of the `tr(…)` calls in one source file. +pub(super) fn messages_in(source: &str) -> BTreeSet { + scan_source(source).into_iter().map(|used| used.key).collect() +} + +/// The literal key of every `tr("…")` call in the app's non-test source. +pub(super) fn source_messages() -> BTreeSet { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + for source in SOURCES { + assert!(root.join(source).is_dir(), "{source} (relative to the crate) is not a folder: the report has nothing to scan"); + source_files(&root.join(source), &mut files); + } + let mut keys = BTreeSet::new(); + for path in files { + let text = fs::read_to_string(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display())); + keys.extend(messages_in(&text)); + } + keys +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_file_named_tests_rs_is_test_code_and_is_not_read() { + // Given a folder with a file of the app and the `tests.rs` of a `#[cfg(test)] mod tests;` + let dir = std::env::temp_dir().join(format!("lightcraft-gaps-{}", std::process::id())); + fs::create_dir_all(dir.join("sub")).unwrap(); + for file in ["app.rs", "sub/tests.rs", "sub/widget.rs"] { + fs::write(dir.join(file), "").unwrap(); + } + // When the source files are listed + let mut files = Vec::new(); + source_files(&dir, &mut files); + let names: Vec = files.iter().map(|path| path.strip_prefix(&dir).unwrap().to_path_buf()).collect(); + fs::remove_dir_all(&dir).unwrap(); + // Then the tests.rs is not among them (compared as paths: the separator is `\` on Windows) + assert_eq!(names, [PathBuf::from("app.rs"), Path::new("sub").join("widget.rs")]); + } + + #[test] + fn the_scanner_reads_literals_and_skips_comments_strings_and_tests() { + let source = r##" +use crate::i18n::tr; +fn f(ui: &mut Ui, on: bool, label: &str) { + ui.label(tr("Plain")); + ui.label(crate::i18n::tr("Qualified")); + ui.label(format!("{}: x", i18n::tr("In a macro"))); + ui.label(Locale::De.tr("Method")); + ui.label(tr("Two \"quoted\" \u{2605} words")); + ui.label(tr("Wrapped \ + line")); + ui.label(tr(r#"Raw "string""#)); + ui.label(tr(if on { "On" } else { "Off" })); + ui.label(tr(label)); + ui.label(tr(&format!("{label}"))); + // tr("In a line comment") + /* tr("In a /* nested */ block comment") */ + let s = "tr(\"In a string\")"; let c = '"'; let r = &'a str; + ui.label(tr("After the char and lifetime")); +} +fn tr(source: &str) -> &str { source } +#[cfg(test)] +mod tests { + fn t() { tr("In a test module"); } +} +#[cfg(test)] +fn only_in_tests() { tr("In a test fn"); } +fn last() { tr("Last"); } +"##; + let keys: Vec<(String, usize)> = scan_source(source).into_iter().map(|used| (used.key, used.line)).collect(); + let keys: Vec<(&str, usize)> = keys.iter().map(|(key, line)| (key.as_str(), *line)).collect(); + assert_eq!( + keys, + vec![ + ("Plain", 4), + ("Qualified", 5), + ("In a macro", 6), + ("Method", 7), + ("Two \"quoted\" \u{2605} words", 8), + ("Wrapped line", 9), + ("Raw \"string\"", 11), + ("On", 12), + ("Off", 12), + ("After the char and lifetime", 18), + ("Last", 27), + ] + ); + } + + #[test] + fn the_scanner_reads_wrapped_calls_and_match_arms_and_skips_only_the_test_item() { + let source = r#" +fn f(m: Mode) { + ui.label(crate::i18n::tr( + "Wrapped by rustfmt", + )); + ui.label(tr(match m { Mode::A => "Arm", Mode::B => { "Braced arm" }, Mode::C => "Last arm" })); + ui.label(tr(match m { Mode::A => other(), Mode::B => "Mixed arm" })); + let r = Renderer { + #[cfg(test)] + built: 0, + after: tr("After a test-only field"), + }; +} +struct S { + #[cfg(test)] + seen: u32 +} +fn g() { tr("After a last test-only field"); } +enum E { + A, + #[cfg(test)] + B { x: u8 }, + #[cfg(test)] + C(u8), + #[cfg(test)] + D = 3, + F, +} +fn after_enum() { tr("After test-only variants"); } +#[cfg(test)] +fn h() -> [u8; 4] { tr("Test only"); [0; 4] } +#[cfg(not(test))] +fn n() { tr("Not a test"); } +"#; + let source = format!("{source}fn crlf() {{\r\n tr(\"Crlf \\\r\n line\");\r\n}}\r\n"); + let keys: Vec = scan_source(&source).into_iter().map(|used| used.key).collect(); + assert_eq!( + keys, + [ + "Wrapped by rustfmt", + "Arm", + "Braced arm", + "Last arm", + "Mixed arm", + "After a test-only field", + "After a last test-only field", + "After test-only variants", + "Not a test", + "Crlf line" + ] + ); + } +} diff --git a/crates/ui-egui/src/i18n/tests.rs b/crates/ui-egui/src/i18n/tests.rs index 77bbc14cb..a7a2f5b9f 100644 --- a/crates/ui-egui/src/i18n/tests.rs +++ b/crates/ui-egui/src/i18n/tests.rs @@ -936,14 +936,40 @@ fn vietnamese_panels_dialogs_and_diacritics_paint_with_bundled_fonts() { } /// The messages each language lacks, which it shows in English. Never fails: `cargo xtask ci` /// runs it with `--nocapture`, so the list lands in the gate log. Plain messages are every key -/// some catalog has plus the labels the UI passes to `tr` from tables (commands, controls, rule -/// fields…); formats are every key some `*-formats.json` has plus every `tr_format!` message in -/// the app's source. An empty translation counts as missing. +/// some catalog has, every `tr("…")` literal in the app's source (`gaps.rs`, wrapped calls and +/// `if`/`match` branches included, so a string no language has yet is listed too) and the +/// labels the UI passes to `tr` from tables (commands, controls, rule fields…); formats are +/// every key some `*-formats.json` has plus every `tr_format!` message in the app's source. An +/// empty translation counts as missing. #[test] fn translation_gaps_report() { let notes = whats_new_lines(); let messages: std::collections::BTreeSet = plain_messages().into_iter().filter(|m| !notes.contains(&m.as_str())).collect(); - let formats = format_messages(); + println!("{}", gaps_text(&messages, &format_messages(), ¬es)); +} + +/// A string only the source parsing finds (a call rustfmt wrapped, the branches of an `if` or a +/// `match`) is in no catalog, and the report still lists it for every language. +#[test] +fn translation_gaps_report_lists_strings_no_catalog_has() { + let found = gaps::messages_in( + "fn f(m: u8) {\n ui.label(crate::i18n::tr(\n \"Only wrapped here\",\n ));\n \ + ui.label(tr(if m == 0 { \"Only in an if\" } else { \"Only in an else\" }));\n \ + ui.label(tr(match m { 1 => \"Only in a match\", _ => \"Only in a last arm\" }));\n}\n", + ); + assert_eq!(found.len(), 5, "{found:?}"); + let text = gaps_text(&found, &Default::default(), &[]); + for language in Locale::ALL.iter().filter(|language| **language != Locale::En) { + let line = text.lines().find(|line| line.starts_with(&format!("{}: ", language.code()))).expect("a line per language"); + assert!(line.contains("5 messages"), "{line}"); + } + for key in &found { + assert!(text.contains(&format!("{key:?}")), "{key}"); + } +} + +/// The report: the totals, then per language its counts and the lists. +fn gaps_text(messages: &std::collections::BTreeSet, formats: &std::collections::BTreeSet, notes: &[&str]) -> String { let quoted = |list: &[&str]| list.iter().map(|m| format!("{m:?}")).collect::>().join(", "); let mut out = format!("Translation gaps, shown in English ({} messages, {} formats, {} What's New lines):\n", messages.len(), formats.len(), notes.len()); @@ -967,11 +993,12 @@ fn translation_gaps_report() { out.push_str(&format!(" formats: {}\n", quoted(&missing_formats))); } } - println!("{out}"); + out } /// Every plain message: the keys of every catalog (but contextual entries, which a language adds -/// only where it needs one) and the labels the UI translates from tables. +/// only where it needs one), the `tr("…")` literals in the source and the labels the UI +/// translates from tables. fn plain_messages() -> std::collections::BTreeSet { use lightcraft_catalog::rules; let mut labels: Vec<&str> = lightcraft_engine::command_specs().iter().map(|spec| spec.label).collect(); @@ -1013,6 +1040,7 @@ fn plain_messages() -> std::collections::BTreeSet { .flat_map(|language| language.catalog().keys()) .filter(|key| !key.starts_with(&contextual)) .cloned() + .chain(gaps::source_messages()) .chain(labels.into_iter().map(str::to_string)) .collect() } diff --git a/docs/localization.md b/docs/localization.md index b878a5de2..36a28d544 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -95,8 +95,12 @@ cargo test -p lightcraft-ui-egui --lib -- --exact i18n::tests::translation_gaps_ The tests fail on a broken catalog (malformed JSON, a repeated key, placeholders that differ from the source), never on a missing translation. The last command is the translation gaps report: per language, the count and list of plain and format messages it lacks and the number of untranslated -What's New lines. It always passes, and `cargo xtask ci` prints it after its test step. The glyph -tests are skipped without `CRAFT_FONTS_DIR` (there are no CJK faces to check then). +What's New lines. The plain messages include every `tr("…")` string in the app's source, whether +or not some catalog has it yet: the report reads calls rustfmt wrapped over several lines and the +literal branches of `tr(if …)` and `tr(match …)`, and skips tests and comments. A string that +reaches `tr` through a variable is listed only when it is a catalog key or in one of the label +tables the report reads. It always passes, and `cargo xtask ci` prints it after its test step. The +glyph tests are skipped without `CRAFT_FONTS_DIR` (there are no CJK faces to check then). To look at a language, render it headless: ```sh