#6005: merge part A (missing glyphs)

crates/layout/src/display/lines.rs

@@ -846,7 +846,7 @@ fn section_break_mark(start: SectionStart, x0: f32, x1: f32, base: f32, size: f3
 fn equation(items: &[MItem], x: f32, base: f32, alpha: f32, screen: bool, out: &mut Vec<Draw>) {
     for it in items {
         match it {
-            MItem::Glyphs { face, size, color, synth_bold, synth_italic, glyphs, text } => out.push(Draw::Glyphs {
+            MItem::Glyphs { face, size, color, synth_bold, synth_italic, glyphs, text, ranges } => out.push(Draw::Glyphs {
                 face: *face,
                 size: *size,
                 glyphs: glyphs.iter().map(|(g, gx, gy)| (*g, x + gx, base - gy)).collect(),
@@ -856,8 +856,7 @@ fn equation(items: &[MItem], x: f32, base: f32, alpha: f32, screen: bool, out: &
                 synth_italic: *synth_italic,
                 text: text.clone(),
                 link: None,
-                // Equation text maps back through the font (no per-glyph ranges from the math layout).
-                ranges: Vec::new(),
+                ranges: ranges.clone(),
                 text_bg: None,
                 x_scale: 1.0,
             }),

crates/layout/src/display/objects.rs

@@ -307,6 +307,7 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) ->
     let mut out = Vec::with_capacity(runs.len());
     for (face, s, shaped) in runs {
         let k = scale(face);
+        let ranges = crate::missing::cluster_ranges(s, &shaped);
         let mut glyphs = Vec::with_capacity(shaped.len());
         for g in &shaped {
             glyphs.push((g.gid, pen + g.x_offset as f32 * k, baseline - g.y_offset as f32 * k));
@@ -324,7 +325,7 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) ->
             synth_italic: false,
             text: s.to_string(),
             link: None,
-            ranges: Vec::new(),
+            ranges,
             text_bg: None,
             x_scale: 1.0,
         });

crates/layout/src/lib.rs

@@ -46,6 +46,7 @@ mod ctx;
 mod floats;
 mod flow;
 mod headers;
+mod missing;
 mod notes;
 mod page;
 mod pass;
@@ -70,6 +71,7 @@ use headers::MAX_HEADER_ITEMS;
 #[cfg(test)]
 use headers::placed_cost;
 use headers::placed_cost_by;
+pub use missing::{MAX_MISSING_GLYPHS, MissingGlyphs};
 pub use notes::DECOR_STORY;
 pub(crate) use notes::NOTE_SEP_W;
 use notes::{MAX_NOTE_PASSES, footnote_pages};

crates/layout/src/math.rs

@@ -41,6 +41,8 @@ pub enum MItem {
         /// Glyph id, x, y (baseline, up).
         glyphs: Vec<(u32, f32, f32)>,
         text: String,
+        /// Byte range of `text` each glyph shows (empty when `text` is: a glyph picked by id).
+        ranges: Vec<std::ops::Range<usize>>,
     },
     /// A filled rectangle; `y` is its bottom edge.
     Rect {

crates/layout/src/math/glyphs.rs

@@ -103,7 +103,16 @@ impl Ctx {
             b.glyph = Some((face, g, attach));
         }
         if !glyphs.is_empty() {
-            b.items.push(MItem::Glyphs { face, size, color, synth_bold: synth.0, synth_italic: synth.1, glyphs, text: text.to_string() });
+            b.items.push(MItem::Glyphs {
+                face,
+                size,
+                color,
+                synth_bold: synth.0,
+                synth_italic: synth.1,
+                glyphs,
+                text: text.to_string(),
+                ranges: crate::missing::cluster_ranges(text, &shaped),
+            });
         }
         b
     }
@@ -132,7 +141,16 @@ impl Ctx {
         if let Some(t) = &self.t {
             b.ic = t.italics.get(&gid).copied().unwrap_or(0.0) * k;
         }
-        b.items.push(MItem::Glyphs { face, size, color, synth_bold: false, synth_italic: false, glyphs: vec![(gid, 0.0, 0.0)], text: String::new() });
+        b.items.push(MItem::Glyphs {
+            face,
+            size,
+            color,
+            synth_bold: false,
+            synth_italic: false,
+            glyphs: vec![(gid, 0.0, 0.0)],
+            text: String::new(),
+            ranges: Vec::new(),
+        });
         b
     }
 
@@ -227,7 +245,16 @@ impl Ctx {
         } else {
             b.w = pos * k;
         }
-        b.items.push(MItem::Glyphs { face: self.face, size, color: self.color, synth_bold: false, synth_italic: false, glyphs, text: String::new() });
+        b.items.push(MItem::Glyphs {
+            face: self.face,
+            size,
+            color: self.color,
+            synth_bold: false,
+            synth_italic: false,
+            glyphs,
+            text: String::new(),
+            ranges: Vec::new(),
+        });
         Some(b)
     }
 

crates/layout/src/missing.rs

@@ -0,0 +1,130 @@
+//! The characters the output would draw as an empty box ([`DocLayout::missing_glyphs`]): glyph
+//! id 0 in a [`Draw::Glyphs`] of the display list, traced back to its character through the
+//! run's `text` and per-glyph `ranges`. Backs strict export (#6005).
+
+use std::collections::BTreeMap;
+use std::ops::{Range, RangeInclusive};
+
+use wordcraft_doc::Document;
+use wordcraft_fonts::ShapedGlyph;
+
+use crate::DocLayout;
+use crate::display::{DisplayOptions, Draw, page_display};
+use crate::para::draws_something;
+
+/// Most distinct code points [`DocLayout::missing_glyphs`] lists: a hostile file full of unmapped
+/// characters can't make the report unbounded.
+pub const MAX_MISSING_GLYPHS: usize = 256;
+
+/// The missing glyphs of a render: characters it draws that no face in the font fallback chain can
+/// draw (glyph id 0, an empty box).
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct MissingGlyphs {
+    /// (code point, times drawn), sorted by code point, at most [`MAX_MISSING_GLYPHS`] long.
+    pub chars: Vec<(char, u64)>,
+    /// More than [`MAX_MISSING_GLYPHS`] distinct code points were missing: the rest aren't listed.
+    pub truncated: bool,
+}
+
+impl MissingGlyphs {
+    /// Nothing is missing.
+    pub fn is_empty(&self) -> bool {
+        self.chars.is_empty() && !self.truncated
+    }
+}
+
+impl DocLayout {
+    /// The characters drawn as glyph id 0 on `pages` (0-based, inclusive; `None` is every page),
+    /// from the display list the PDF and PNG exports draw with `opts`: body, headers, footers,
+    /// text boxes, notes, table cells, charts, diagrams and equations. A character counts once per
+    /// glyph drawn for it. Default-ignorable and control characters never count (they draw
+    /// nothing), and a Private Use Area character counts only if it really drew as glyph 0.
+    /// Pages past the end are ignored.
+    pub fn missing_glyphs(&self, doc: &Document, opts: &DisplayOptions, pages: Option<RangeInclusive<usize>>) -> MissingGlyphs {
+        let mut found = Found::default();
+        for (i, page) in self.pages.iter().enumerate() {
+            if pages.as_ref().is_some_and(|r| !r.contains(&i)) {
+                continue;
+            }
+            for d in page_display(doc, page, opts) {
+                found.walk(&d);
+            }
+        }
+        MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated }
+    }
+}
+
+#[derive(Default)]
+struct Found {
+    counts: BTreeMap<char, u64>,
+    truncated: bool,
+}
+
+impl Found {
+    fn walk(&mut self, d: &Draw) {
+        match d {
+            Draw::Glyphs { face, glyphs, text, ranges, .. } => {
+                let one_each = ranges.is_empty() && text.chars().count() == glyphs.len();
+                let mut chars = text.chars();
+                for (k, (gid, ..)) in glyphs.iter().enumerate() {
+                    // A run without ranges (a list label) maps one character to one glyph, when
+                    // the counts agree; else it can't be traced and isn't counted.
+                    let own = chars.next();
+                    if *gid != 0 {
+                        continue;
+                    }
+                    let shown: Vec<char> = if one_each {
+                        own.into_iter().collect()
+                    } else {
+                        ranges.get(k).and_then(|r| text.get(r.clone())).map(|t| t.chars().collect()).unwrap_or_default()
+                    };
+                    let drawn: Vec<char> = shown.into_iter().filter(|c| draws_something(*c)).collect();
+                    // A cluster of several characters drew one empty box: it is the one the face
+                    // lacks, not the letter beside it.
+                    let lacking: Vec<char> = drawn.iter().copied().filter(|c| !face.covers(*c)).collect();
+                    for c in if lacking.is_empty() { drawn } else { lacking } {
+                        self.count(c);
+                    }
+                }
+            }
+            Draw::Figure { draws: items, .. } | Draw::Turned { items, .. } | Draw::Rotated { items, .. } | Draw::Clip { items, .. } => {
+                for d in items {
+                    self.walk(d);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    fn count(&mut self, c: char) {
+        if let Some(n) = self.counts.get_mut(&c) {
+            *n = n.saturating_add(1);
+        } else if self.counts.len() < MAX_MISSING_GLYPHS {
+            self.counts.insert(c, 1);
+        } else {
+            self.truncated = true;
+        }
+    }
+}
+
+/// The byte range of `text` each of `shaped` shows: from its cluster to the next cluster start in
+/// the text (or its end), whichever order the glyphs are in. A ligature's glyph covers all its
+/// characters; the glyphs of one cluster share its range. Ranges always lie on character
+/// boundaries of `text`.
+pub(crate) fn cluster_ranges(text: &str, shaped: &[ShapedGlyph]) -> Vec<Range<usize>> {
+    let mut starts: Vec<usize> = shaped.iter().map(|g| g.cluster.min(text.len())).filter(|b| text.is_char_boundary(*b)).collect();
+    starts.sort_unstable();
+    starts.dedup();
+    shaped
+        .iter()
+        .map(|g| {
+            let from = g.cluster.min(text.len());
+            let from = if text.is_char_boundary(from) { from } else { (0..from).rev().find(|b| text.is_char_boundary(*b)).unwrap_or(0) };
+            let to = match starts.binary_search(&from) {
+                Ok(i) => starts.get(i + 1).copied().unwrap_or(text.len()),
+                Err(_) => text.len(),
+            };
+            from..to.max(from)
+        })
+        .collect()
+}

crates/layout/src/tests.rs

@@ -13,6 +13,7 @@ mod float_top;
 mod grid;
 mod ligature;
 pub(crate) mod linear;
+mod missing_glyphs;
 mod nbsp;
 mod nested_cant_split;
 mod numeral;

crates/layout/src/tests/missing_glyphs.rs

@@ -0,0 +1,252 @@
+//! [`DocLayout::missing_glyphs`] (strict export, #6005): glyph id 0 in the display list, named by
+//! its character, wherever the text is drawn.
+
+use std::sync::Arc;
+
+use wordcraft_doc::graphic::{Graphic, GraphicItem, GraphicKind, TextAlign};
+use wordcraft_doc::para::InlineObject;
+use wordcraft_doc::props::Rgb;
+use wordcraft_doc::{Block, Document, Path, Pos, StoryRef, Table};
+use wordcraft_fonts::FontDb;
+
+use super::{footnote, lay, page_float, text_box_at};
+use crate::display::DisplayOptions;
+use crate::{DocLayout, MAX_MISSING_GLYPHS, MissingGlyphs};
+
+/// Code points in unassigned plane 4 that no face of the host covers, checked against the body
+/// font and the whole fallback chain (so the test never guesses).
+fn uncovered(n: usize) -> Vec<char> {
+    let db = FontDb::global();
+    let primary = wordcraft_fonts::word::resolve(&Document::default().settings.minor_font, false, false).face;
+    let out: Vec<char> = (0x40000u32..0x4FFF0)
+        .filter_map(char::from_u32)
+        .filter(|c| {
+            !primary.covers(*c) && db.fallback_styled(*c, primary.id(), false, false).is_none() && db.fallback_for(*c, primary.id()).is_none()
+        })
+        .take(n)
+        .collect();
+    assert_eq!(out.len(), n, "the host has faces covering all of plane 4");
+    out
+}
+
+fn missing(d: &Document, l: &DocLayout) -> MissingGlyphs {
+    l.missing_glyphs(d, &DisplayOptions::default(), None)
+}
+
+fn chars(m: &MissingGlyphs) -> Vec<char> {
+    m.chars.iter().map(|(c, _)| *c).collect()
+}
+
+fn hf_part(d: &mut Document, kind: wordcraft_doc::PartKind, text: &str) -> u32 {
+    d.add_part(kind, vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(text, Default::default()))])
+}
+
+fn chart_with(d: &mut Document, label: &str) {
+    let items = vec![GraphicItem::Text {
+        rect: [0.0, 100.0, 216.0, 20.0],
+        text: label.into(),
+        size: 10.0,
+        color: Rgb::BLACK,
+        bold: false,
+        align: TextAlign::Center,
+        font: None,
+    }];
+    let graphic = Graphic { kind: GraphicKind::Chart, items, ..Default::default() };
+    let obj = InlineObject::Graphic { w: 216.0, h: 144.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) };
+    d.insert_object(&Pos::body(0, 0), obj, &Default::default()).unwrap();
+}
+
+fn equation_with(d: &mut Document, linear: &str) {
+    let math = wordcraft_doc::math::Math { nodes: wordcraft_doc::math::parse_linear(linear), ..Default::default() };
+    let n = d.para_mut(StoryRef::Body, &Path::top(0)).unwrap().len();
+    d.para_mut(StoryRef::Body, &Path::top(0))
+        .unwrap()
+        .insert_object(n, InlineObject::Equation { linear: linear.into(), display: false, math }, &Default::default())
+        .unwrap();
+}
+
+#[test]
+fn nothing_is_missing_in_plain_text() {
+    let d = Document::from_text("Hello, world.");
+    assert!(missing(&d, &lay(&d)).is_empty());
+}
+
+#[test]
+fn a_character_no_face_covers_is_counted_once_per_occurrence_in_the_body() {
+    let c = uncovered(1)[0];
+    let d = Document::from_text(&format!("a{c}b{c}{c}c"));
+    let m = missing(&d, &lay(&d));
+    assert_eq!(m, MissingGlyphs { chars: vec![(c, 3)], truncated: false });
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_header() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Body");
+    d.last_section.headers.default = Some(hf_part(&mut d, wordcraft_doc::PartKind::Header, &format!("H{c}{c}")));
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 2)]);
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_footer() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Body");
+    d.last_section.footers.default = Some(hf_part(&mut d, wordcraft_doc::PartKind::Footer, &format!("F{c}")));
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]);
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_footnote() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Body text.");
+    footnote(&mut d, &Pos::body(0, 4), &format!("Note {c}"));
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]);
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_text_box() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Body text.");
+    text_box_at(&mut d, &Pos::body(0, 0), &format!("Box {c}{c}{c}"), 120.0, 40.0, page_float(300.0, 300.0));
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 3)]);
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_table_cell() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("before");
+    let mut t = Table::new(1, 2, 300.0);
+    t.rows[0].cells[1].blocks = vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(&format!("cell {c}"), Default::default()))];
+    d.insert_block(StoryRef::Body, &Path::top(1), Block::Table(t)).unwrap();
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]);
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_a_chart_label() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Chart");
+    chart_with(&mut d, &format!("Q{c}Q{c}"));
+    assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 2)]);
+}
+
+#[test]
+fn a_chart_label_run_names_each_glyphs_characters() {
+    use crate::display::Draw;
+    let mut d = Document::from_text("Chart");
+    chart_with(&mut d, "Q1 total");
+    let l = lay(&d);
+    fn runs(d: &Draw, out: &mut Vec<(String, usize, usize)>) {
+        match d {
+            Draw::Glyphs { text, glyphs, ranges, .. } if text == "Q1 total" || !ranges.is_empty() => {
+                out.push((text.clone(), glyphs.len(), ranges.len()));
+                assert!(ranges.iter().all(|r| text.get(r.clone()).is_some()), "ranges on character boundaries");
+            }
+            Draw::Figure { draws, .. } => draws.iter().for_each(|x| runs(x, out)),
+            _ => {}
+        }
+    }
+    let mut found = Vec::new();
+    for x in crate::display::page_display(&d, &l.pages[0], &DisplayOptions::default()) {
+        runs(&x, &mut found);
+    }
+    assert!(found.iter().any(|(t, g, r)| t == "Q1 total" && g == r), "{found:?}");
+}
+
+#[test]
+fn a_missing_glyph_is_found_in_an_equation() {
+    let c = uncovered(1)[0];
+    let mut d = Document::from_text("Eq: ");
+    equation_with(&mut d, &format!("x+{c}"));
+    let m = missing(&d, &lay(&d));
+    assert_eq!(m.chars, vec![(c, 1)], "{m:?}");
+}
+
+#[test]
+fn names_in_many_scripts_report_nothing() {
+    if std::env::var_os("CRAFT_FONTS_DIR").is_none() {
+        eprintln!("CRAFT_FONTS_DIR unset: the multilingual faces aren't available, skipping");
+        return;
+    }
+    let names = ["Łukasz Żmijewski", "Gheorghe Ștefănescu", "Krejčí", "Şahın Gül", "Llŷr Grŵn", "王小明", "김민수", "สมชาย ใจดี", "राम शर्मा"];
+    let d = Document::from_text(&names.join("\n"));
+    let m = missing(&d, &lay(&d));
+    assert!(m.is_empty(), "{m:?}");
+}
+
+#[test]
+fn joiners_and_variation_selectors_are_never_counted() {
+    let d = Document::from_text("a\u{200D}b\u{200C}c\u{FE0F}d\u{FE0E}e\u{00AD}f\u{200E}g\u{2028}h\ti");
+    assert!(missing(&d, &lay(&d)).is_empty());
+    // A ZWJ emoji sequence: the joiners and selectors in it never appear, whatever the host's
+    // emoji face covers.
+    let d = Document::from_text("\u{1F469}\u{200D}\u{1F4BB} \u{2764}\u{FE0F} \u{1F44D}\u{1F3FD}");
+    let m = missing(&d, &lay(&d));
+    assert!(!chars(&m).iter().any(|c| matches!(c, '\u{200D}' | '\u{FE0F}' | '\u{FE0E}')), "{m:?}");
+}
+
+#[test]
+fn a_selector_beside_a_missing_character_is_not_counted_with_it() {
+    let c = uncovered(1)[0];
+    let d = Document::from_text(&format!("{c}\u{FE0F}"));
+    assert_eq!(chars(&missing(&d, &lay(&d))), vec![c]);
+}
+
+#[test]
+fn the_list_is_sorted_and_capped_at_256_distinct_code_points() {
+    let all = uncovered(MAX_MISSING_GLYPHS + 44);
+    let text: String = all.iter().rev().collect();
+    let d = Document::from_text(&text);
+    let m = missing(&d, &lay(&d));
+    assert!(m.truncated);
+    assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS);
+    assert!(m.chars.windows(2).all(|w| w[0].0 < w[1].0), "sorted by code point");
+    // Exactly the cap's worth is no truncation.
+    let d = Document::from_text(&all.iter().take(MAX_MISSING_GLYPHS).collect::<String>());
+    let m = missing(&d, &lay(&d));
+    assert!(!m.truncated);
+    assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS);
+}
+
+#[test]
+fn the_page_filter_selects_pages() {
+    let [x, y] = <[char; 2]>::try_from(uncovered(2)).unwrap();
+    let d = Document::from_text(&format!("first {x}\n{}last {y}", "filler line\n".repeat(150)));
+    let l = lay(&d);
+    let last = l.pages.len() - 1;
+    assert!(last >= 1, "the document runs over pages");
+    let on = |r| l.missing_glyphs(&d, &DisplayOptions::default(), r);
+    assert_eq!(on(None).chars, vec![(x.min(y), 1), (x.max(y), 1)]);
+    assert_eq!(on(Some(0..=0)).chars, vec![(x, 1)]);
+    assert_eq!(on(Some(last..=last)).chars, vec![(y, 1)]);
+    assert!(on(Some(last + 1..=last + 9)).is_empty());
+    #[allow(clippy::reversed_empty_ranges)]
+    let backwards = Some(1..=0);
+    assert!(on(backwards).is_empty());
+}
+
+#[test]
+fn an_equation_run_names_each_glyphs_characters() {
+    use crate::display::Draw;
+    let mut d = Document::from_text("Eq: ");
+    equation_with(&mut d, "x+ab");
+    let l = lay(&d);
+    fn runs(d: &Draw, out: &mut Vec<(String, usize, usize)>) {
+        match d {
+            Draw::Glyphs { text, glyphs, ranges, .. } if !text.is_empty() => {
+                assert!(ranges.iter().all(|r| text.get(r.clone()).is_some()), "ranges on character boundaries");
+                out.push((text.clone(), glyphs.len(), ranges.len()));
+            }
+            Draw::Figure { draws, .. } | Draw::Clip { items: draws, .. } | Draw::Rotated { items: draws, .. } => {
+                draws.iter().for_each(|x| runs(x, out))
+            }
+            _ => {}
+        }
+    }
+    let mut found = Vec::new();
+    for x in crate::display::page_display(&d, &l.pages[0], &DisplayOptions::default()) {
+        runs(&x, &mut found);
+    }
+    // The equation's runs (the body text "Eq: " is a run too) each carry one range per glyph.
+    assert!(found.iter().any(|(t, g, _)| t.contains('\u{1D44E}') && *g == 2), "{found:?}");
+    assert!(found.iter().all(|(_, g, r)| g == r), "{found:?}");
+}