Fix #4940: text is shaped in its language

Characters get a language (CharStyle::language, a BCP 47 tag of one of
Document Setup's languages; none for the default, English: USA, saved
only when set). The shaper sets it on the HarfRust buffer, so fonts use
their language forms (OpenType locl): Turkish keeps fi apart and the dot
on its i. Runs in different languages are shaped apart. New type takes
the document's language.

crates/doc/src/setup.rs

@@ -143,34 +143,55 @@ impl Quotes {
     }
 }
 
-/// Document languages and the quote characters each uses (choosing a language in Document Setup
-/// also picks its quotes).
-pub const LANGUAGES: &[(&str, Quotes)] = &[
-    ("English: USA", Quotes::ENGLISH),
-    ("English: UK", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
-    ("German", Quotes { double: ['„', '“'], single: ['‚', '‘'] }),
-    ("German: Swiss", Quotes { double: ['«', '»'], single: ['‹', '›'] }),
-    ("French", Quotes { double: ['«', '»'], single: ['‹', '›'] }),
-    ("Italian", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
-    ("Spanish", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
-    ("Catalan", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
-    ("Portuguese", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
-    ("Dutch", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
-    ("Danish", Quotes { double: ['»', '«'], single: ['›', '‹'] }),
-    ("Swedish", Quotes { double: ['”', '”'], single: ['’', '’'] }),
-    ("Finnish", Quotes { double: ['”', '”'], single: ['’', '’'] }),
-    ("Norwegian", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
-    ("Polish", Quotes { double: ['„', '”'], single: ['‚', '’'] }),
-    ("Czech", Quotes { double: ['„', '“'], single: ['‚', '‘'] }),
-    ("Russian", Quotes { double: ['«', '»'], single: ['„', '“'] }),
-    ("Turkish", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
-    ("Japanese", Quotes { double: ['「', '」'], single: ['『', '』'] }),
-    ("Chinese", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
+/// Document and character languages: the name, the BCP 47 tag text is shaped in (so that fonts use
+/// their language's forms, OpenType `locl`) and the quote characters each uses (choosing a
+/// language in Document Setup also picks its quotes). The first is the default.
+pub const LANGUAGES: &[(&str, &str, Quotes)] = &[
+    ("English: USA", "en-US", Quotes::ENGLISH),
+    ("English: UK", "en-GB", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
+    ("German", "de", Quotes { double: ['„', '“'], single: ['‚', '‘'] }),
+    ("German: Swiss", "de-CH", Quotes { double: ['«', '»'], single: ['‹', '›'] }),
+    ("French", "fr", Quotes { double: ['«', '»'], single: ['‹', '›'] }),
+    ("Italian", "it", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
+    ("Spanish", "es", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
+    ("Catalan", "ca", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
+    ("Portuguese", "pt", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
+    ("Dutch", "nl", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
+    ("Danish", "da", Quotes { double: ['»', '«'], single: ['›', '‹'] }),
+    ("Swedish", "sv", Quotes { double: ['”', '”'], single: ['’', '’'] }),
+    ("Finnish", "fi", Quotes { double: ['”', '”'], single: ['’', '’'] }),
+    ("Norwegian", "nb", Quotes { double: ['«', '»'], single: ['‘', '’'] }),
+    ("Polish", "pl", Quotes { double: ['„', '”'], single: ['‚', '’'] }),
+    ("Czech", "cs", Quotes { double: ['„', '“'], single: ['‚', '‘'] }),
+    ("Russian", "ru", Quotes { double: ['«', '»'], single: ['„', '“'] }),
+    ("Turkish", "tr", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
+    ("Japanese", "ja", Quotes { double: ['「', '」'], single: ['『', '』'] }),
+    ("Chinese", "zh", Quotes { double: ['“', '”'], single: ['‘', '’'] }),
 ];
 
 /// The quotes of a [`LANGUAGES`] entry (case-insensitive).
 pub fn language_quotes(language: &str) -> Option<Quotes> {
-    LANGUAGES.iter().find(|(l, _)| l.eq_ignore_ascii_case(language)).map(|(_, q)| *q)
+    LANGUAGES.iter().find(|(l, _, _)| l.eq_ignore_ascii_case(language)).map(|(_, _, q)| *q)
+}
+
+/// The [`LANGUAGES`] entry named or tagged `language` (case-insensitive) as a character's
+/// language ([`crate::CharStyle::language`]) stores it: its tag, or `None` for the default (the
+/// first entry); `Err` for a language not in the list.
+pub fn char_language(language: &str) -> Result<Option<&'static str>, String> {
+    let i = LANGUAGES
+        .iter()
+        .position(|(n, t, _)| n.eq_ignore_ascii_case(language) || t.eq_ignore_ascii_case(language))
+        .ok_or_else(|| format!("unknown language `{language}` (one of {})", LANGUAGES.iter().map(|(n, _, _)| *n).collect::<Vec<_>>().join(", ")))?;
+    Ok(LANGUAGES.get(i).filter(|_| i > 0).map(|(_, t, _)| *t))
+}
+
+/// The name of the language a character's stored `language` tag stands for: the default's for
+/// none, `None` for a tag not in [`LANGUAGES`] (from a file).
+pub fn char_language_name(language: Option<&str>) -> Option<&'static str> {
+    match language {
+        None => LANGUAGES.first().map(|(n, _, _)| *n),
+        Some(tag) => LANGUAGES.iter().find(|(_, t, _)| t.eq_ignore_ascii_case(tag)).map(|(n, _, _)| *n),
+    }
 }
 
 /// The transparency flattener preset Document Setup uses unless another is chosen.
@@ -329,6 +350,24 @@ impl DocSetup {
 mod tests {
     use super::*;
 
+    /// Characters store a language of the list by its tag, the default as none.
+    #[test]
+    fn char_languages_are_stored_by_tag() {
+        assert_eq!(char_language("Turkish"), Ok(Some("tr")));
+        assert_eq!(char_language("TR"), Ok(Some("tr")));
+        assert_eq!(char_language("english: usa"), Ok(None));
+        assert_eq!(char_language("en-US"), Ok(None));
+        assert!(char_language("Klingon").is_err() && char_language("").is_err());
+        assert_eq!(
+            (char_language_name(None), char_language_name(Some("de")), char_language_name(Some("x-unknown"))),
+            (Some("English: USA"), Some("German"), None)
+        );
+        // Every entry has a tag of its own, and a name of its own.
+        for (i, (n, t, _)) in LANGUAGES.iter().enumerate() {
+            assert!(LANGUAGES.iter().skip(i + 1).all(|(n2, t2, _)| n2 != n && t2 != t), "{n} {t}");
+        }
+    }
+
     #[test]
     fn quotes_open_and_close_by_context() {
         let mut prev = None;

crates/doc/src/text/char_style.rs

@@ -114,6 +114,12 @@ pub struct CharStyle {
     /// <https://helpx.adobe.com/illustrator/using/hyphenation-line-breaks.html>
     #[serde(default, rename = "noBreak", skip_serializing_if = "crate::skip::is_default")]
     pub no_break: bool,
+    /// Language (Character panel): the BCP 47 tag of one of [`crate::setup::LANGUAGES`] these
+    /// characters are shaped in, so that the font uses that language's forms (OpenType `locl`).
+    /// `None`: the default language, English: USA, shaped without a language. New type takes the
+    /// document's language (Document Setup). Saved only when set.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub language: Option<String>,
 }
 
 /// The numeral system digits are drawn in ([`CharStyle::digits`]): as typed, or every digit of
@@ -297,6 +303,7 @@ impl Default for CharStyle {
             small_caps: None,
             digits: Digits::Default,
             no_break: false,
+            language: None,
         }
     }
 }

crates/doc/src/text/tests.rs

@@ -60,6 +60,18 @@ fn digits_map_between_the_numeral_systems() {
     assert_eq!(serde_json::from_value::<CharStyle>(serde_json::to_value(&farsi).unwrap()).unwrap(), farsi);
 }
 
+/// Language is saved only when set; styles written before it existed read as the default.
+#[test]
+fn language_is_saved_only_when_set() {
+    let st = CharStyle::default();
+    let old = serde_json::to_value(&st).unwrap();
+    assert!(old.get("language").is_none());
+    assert_eq!(serde_json::from_value::<CharStyle>(old).unwrap().language, None);
+    let turkish = CharStyle { language: Some("tr".into()), ..CharStyle::default() };
+    assert_eq!(serde_json::to_value(&turkish).unwrap()["language"], "tr");
+    assert_eq!(serde_json::from_value::<CharStyle>(serde_json::to_value(&turkish).unwrap()).unwrap(), turkish);
+}
+
 /// Auto leading follows the paragraph's Auto Leading; a set leading doesn't.
 #[test]
 fn paragraph_effective_leading_follows_auto_leading() {

crates/engine/src/cmd/create.rs

@@ -811,7 +811,8 @@ pub(crate) fn new_area_fit(s: &Session, p: &Value, c: &str) -> Result<vectorcraf
 
 /// The character style new type gets: `size`, `font`, `style` and `color` from `p` (an RGB
 /// `color` in a CMYK document as [`super::paint::new_color`] stores it), else the defaults,
-/// filled with the current fill unless that is None or white (black then).
+/// filled with the current fill unless that is None or white (black then), in the document's
+/// language.
 pub(crate) fn new_type_style(s: &Session, p: &Value) -> CharStyle {
     let mut style = CharStyle::default();
     if let Some(sz) = p.get("size").and_then(Value::as_f64) {
@@ -831,5 +832,7 @@ pub(crate) fn new_type_style(s: &Session, p: &Value) -> CharStyle {
         None if !s.paint.fill.is_none() && s.paint.fill != vectorcraft_color::Paint::solid(vectorcraft_color::Color::WHITE) => s.paint.fill.clone(),
         None => vectorcraft_color::Paint::solid(vectorcraft_color::Color::BLACK),
     };
+    // In the document's language (Document Setup), none for the default.
+    style.language = s.doc().ok().and_then(|d| vectorcraft_doc::setup::char_language(&d.doc.setup.language).ok().flatten()).map(str::to_string);
     style
 }

crates/engine/src/cmd/docsetup.rs

@@ -127,8 +127,8 @@ fn apply(s: &mut DocSetup, units: &mut vectorcraft_doc::Unit, k: &str, v: &Value
         "discardWhiteOverprint" => s.discard_white_overprint = flag(k, v)?,
         "language" => {
             let l = v.as_str().unwrap_or("");
-            let (name, quotes) = LANGUAGES.iter().find(|(n, _)| n.eq_ignore_ascii_case(l)).ok_or_else(|| {
-                bad(C, format!("unknown language `{l}` (one of {})", LANGUAGES.iter().map(|(n, _)| *n).collect::<Vec<_>>().join(", ")))
+            let (name, _, quotes) = LANGUAGES.iter().find(|(n, _, _)| n.eq_ignore_ascii_case(l)).ok_or_else(|| {
+                bad(C, format!("unknown language `{l}` (one of {})", LANGUAGES.iter().map(|(n, _, _)| *n).collect::<Vec<_>>().join(", ")))
             })?;
             s.language = name.to_string();
             s.quotes = *quotes;

crates/text/src/features.rs

@@ -295,4 +295,22 @@ mod tests {
         assert!(OtFeatures::default().to_tags().is_empty());
         assert!(OtFeatures::known_tag("-onum") && !OtFeatures::known_tag("zzzz"));
     }
+
+    /// The characters' language reaches the shaper (#4940): Source Serif 4's Turkish forms
+    /// (`locl`) keep f and i apart, and the dot on the i, where English ligates them. Runs in
+    /// different languages are shaped apart, each in its own.
+    #[test]
+    fn language_picks_the_fonts_forms() {
+        let st =
+            |language: Option<&str>| CharStyle { font_family: "Source Serif 4".into(), language: language.map(str::to_string), ..Default::default() };
+        let count = |language| layout(FontDb::global(), &TextObject::point(Point::ZERO, "fi", st(language))).glyphs.len();
+        assert_eq!((count(None), count(Some("en-US")), count(Some("tr"))), (1, 1, 2));
+        // A tag that isn't one (from a file) shapes without a language.
+        assert_eq!((count(Some("tr; DROP")), count(Some(""))), (1, 1));
+        let mut t = TextObject::point(Point::ZERO, "", st(None));
+        t.runs = vec![vectorcraft_doc::TextRun::new("fi", st(Some("tr"))), vectorcraft_doc::TextRun::new("fi", st(None))];
+        let glyphs = layout(FontDb::global(), &t).glyphs;
+        let per_run: Vec<usize> = (0..2).map(|r| glyphs.iter().filter(|g| g.run == r).count()).collect();
+        assert_eq!(per_run, [2, 1]);
+    }
 }

crates/text/src/shape.rs

@@ -715,6 +715,7 @@ fn shapes_alike(a: &CharStyle, b: &CharStyle) -> bool {
         tate_chu_yoko,
         digits,
         no_break,
+        language,
     } = a;
     *font_family == b.font_family
         && *font_style == b.font_style
@@ -731,6 +732,7 @@ fn shapes_alike(a: &CharStyle, b: &CharStyle) -> bool {
         && *tate_chu_yoko == b.tate_chu_yoko
         && *digits == b.digits
         && *no_break == b.no_break
+        && *language == b.language
         && ligatures_suppressed_by(*tracking) == ligatures_suppressed_by(b.tracking)
 }
 
@@ -922,6 +924,12 @@ fn shape_segment(text: &str, context: &Range<usize>, seg: &Segment, feats: &OtFe
 
     // Shape the segment as if all of it were in `st`, with the features `more` on top.
     let shape_in = |st: &CharStyle, more: &[Feature]| {
+        // A BCP 47 tag is short ASCII; anything else (a file may hold anything) is shaped without one.
+        let language = st
+            .language
+            .as_deref()
+            .filter(|l| l.len() <= 35 && l.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
+            .and_then(|l| l.parse::<harfrust::Language>().ok());
         let mut shaped = Shaped::default();
         let Shaped { raw, proportional, vpal } = &mut shaped;
         let hb_shaped = face.hb().map(|hb| {
@@ -943,6 +951,11 @@ fn shape_segment(text: &str, context: &Range<usize>, seg: &Segment, feats: &OtFe
                 }
                 buf.set_direction(direction);
                 buf.guess_segment_properties();
+                // The characters' language picks the font's forms for it (`locl`: Turkish keeps fi
+                // apart and the dot on its i).
+                if let Some(l) = &language {
+                    buf.set_language(l.clone());
+                }
                 // Harfrust keeps this Unicode context for joining/positional shaping, but does not emit
                 // glyphs for it. This retains context at real style/font segmentation boundaries without
                 // shaping across those boundaries or claiming cross-style ligatures/mark attachment.

crates/ui-egui/src/dialogs/document_setup.rs

@@ -154,7 +154,7 @@ fn transparency(app: &mut VectorcraftApp, ui: &mut egui::Ui, d: &mut Dialog) {
 
 fn typography(ui: &mut egui::Ui, d: &mut Dialog) {
     let t = Tokens::get(ui.ctx());
-    let langs: Vec<(&str, &str)> = LANGUAGES.iter().map(|(l, _)| (*l, *l)).collect();
+    let langs: Vec<(&str, &str)> = LANGUAGES.iter().map(|(l, _, _)| (*l, *l)).collect();
     let before = d.str("language");
     choice(ui, d, "language", tl!("Language:"), &langs);
     // A language brings its quotes.