diff --git a/crates/layout/src/display/lines.rs b/crates/layout/src/display/lines.rs index 059358ad82..3b707b5e85 100644 --- a/crates/layout/src/display/lines.rs +++ b/crates/layout/src/display/lines.rs @@ -846,7 +846,7 @@ fn section_break_mark(start: SectionStart, x0: f32, x1: f32, base: f32, size: f3 fn equation(items: &[MItem], x: f32, base: f32, alpha: f32, screen: bool, out: &mut Vec) { for it in items { match it { - MItem::Glyphs { face, size, color, synth_bold, synth_italic, glyphs, text } => out.push(Draw::Glyphs { + MItem::Glyphs { face, size, color, synth_bold, synth_italic, glyphs, text, ranges } => out.push(Draw::Glyphs { face: *face, size: *size, glyphs: glyphs.iter().map(|(g, gx, gy)| (*g, x + gx, base - gy)).collect(), @@ -856,8 +856,7 @@ fn equation(items: &[MItem], x: f32, base: f32, alpha: f32, screen: bool, out: & synth_italic: *synth_italic, text: text.clone(), link: None, - // Equation text maps back through the font (no per-glyph ranges from the math layout). - ranges: Vec::new(), + ranges: ranges.clone(), text_bg: None, x_scale: 1.0, }), diff --git a/crates/layout/src/display/objects.rs b/crates/layout/src/display/objects.rs index 89211cf41a..ee574e735e 100644 --- a/crates/layout/src/display/objects.rs +++ b/crates/layout/src/display/objects.rs @@ -307,6 +307,7 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) -> let mut out = Vec::with_capacity(runs.len()); for (face, s, shaped) in runs { let k = scale(face); + let ranges = crate::missing::cluster_ranges(s, &shaped); let mut glyphs = Vec::with_capacity(shaped.len()); for g in &shaped { glyphs.push((g.gid, pen + g.x_offset as f32 * k, baseline - g.y_offset as f32 * k)); @@ -324,7 +325,7 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) -> synth_italic: false, text: s.to_string(), link: None, - ranges: Vec::new(), + ranges, text_bg: None, x_scale: 1.0, }); diff --git a/crates/layout/src/lib.rs b/crates/layout/src/lib.rs index 30aaa66e61..3352de98ad 100644 --- a/crates/layout/src/lib.rs +++ b/crates/layout/src/lib.rs @@ -46,6 +46,7 @@ mod ctx; mod floats; mod flow; mod headers; +mod missing; mod notes; mod page; mod pass; @@ -70,6 +71,7 @@ use headers::MAX_HEADER_ITEMS; #[cfg(test)] use headers::placed_cost; use headers::placed_cost_by; +pub use missing::{MAX_MISSING_GLYPHS, MissingGlyphs}; pub use notes::DECOR_STORY; pub(crate) use notes::NOTE_SEP_W; use notes::{MAX_NOTE_PASSES, footnote_pages}; diff --git a/crates/layout/src/math.rs b/crates/layout/src/math.rs index 95070ecd2f..2f71bb5bec 100644 --- a/crates/layout/src/math.rs +++ b/crates/layout/src/math.rs @@ -41,6 +41,8 @@ pub enum MItem { /// Glyph id, x, y (baseline, up). glyphs: Vec<(u32, f32, f32)>, text: String, + /// Byte range of `text` each glyph shows (empty when `text` is: a glyph picked by id). + ranges: Vec>, }, /// A filled rectangle; `y` is its bottom edge. Rect { diff --git a/crates/layout/src/math/glyphs.rs b/crates/layout/src/math/glyphs.rs index 25dfe57b0d..6c6f45c5af 100644 --- a/crates/layout/src/math/glyphs.rs +++ b/crates/layout/src/math/glyphs.rs @@ -103,7 +103,16 @@ impl Ctx { b.glyph = Some((face, g, attach)); } if !glyphs.is_empty() { - b.items.push(MItem::Glyphs { face, size, color, synth_bold: synth.0, synth_italic: synth.1, glyphs, text: text.to_string() }); + b.items.push(MItem::Glyphs { + face, + size, + color, + synth_bold: synth.0, + synth_italic: synth.1, + glyphs, + text: text.to_string(), + ranges: crate::missing::cluster_ranges(text, &shaped), + }); } b } @@ -132,7 +141,16 @@ impl Ctx { if let Some(t) = &self.t { b.ic = t.italics.get(&gid).copied().unwrap_or(0.0) * k; } - b.items.push(MItem::Glyphs { face, size, color, synth_bold: false, synth_italic: false, glyphs: vec![(gid, 0.0, 0.0)], text: String::new() }); + b.items.push(MItem::Glyphs { + face, + size, + color, + synth_bold: false, + synth_italic: false, + glyphs: vec![(gid, 0.0, 0.0)], + text: String::new(), + ranges: Vec::new(), + }); b } @@ -227,7 +245,16 @@ impl Ctx { } else { b.w = pos * k; } - b.items.push(MItem::Glyphs { face: self.face, size, color: self.color, synth_bold: false, synth_italic: false, glyphs, text: String::new() }); + b.items.push(MItem::Glyphs { + face: self.face, + size, + color: self.color, + synth_bold: false, + synth_italic: false, + glyphs, + text: String::new(), + ranges: Vec::new(), + }); Some(b) } diff --git a/crates/layout/src/missing.rs b/crates/layout/src/missing.rs new file mode 100644 index 0000000000..ed3a419cb6 --- /dev/null +++ b/crates/layout/src/missing.rs @@ -0,0 +1,130 @@ +//! The characters the output would draw as an empty box ([`DocLayout::missing_glyphs`]): glyph +//! id 0 in a [`Draw::Glyphs`] of the display list, traced back to its character through the +//! run's `text` and per-glyph `ranges`. Backs strict export (#6005). + +use std::collections::BTreeMap; +use std::ops::{Range, RangeInclusive}; + +use wordcraft_doc::Document; +use wordcraft_fonts::ShapedGlyph; + +use crate::DocLayout; +use crate::display::{DisplayOptions, Draw, page_display}; +use crate::para::draws_something; + +/// Most distinct code points [`DocLayout::missing_glyphs`] lists: a hostile file full of unmapped +/// characters can't make the report unbounded. +pub const MAX_MISSING_GLYPHS: usize = 256; + +/// The missing glyphs of a render: characters it draws that no face in the font fallback chain can +/// draw (glyph id 0, an empty box). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MissingGlyphs { + /// (code point, times drawn), sorted by code point, at most [`MAX_MISSING_GLYPHS`] long. + pub chars: Vec<(char, u64)>, + /// More than [`MAX_MISSING_GLYPHS`] distinct code points were missing: the rest aren't listed. + pub truncated: bool, +} + +impl MissingGlyphs { + /// Nothing is missing. + pub fn is_empty(&self) -> bool { + self.chars.is_empty() && !self.truncated + } +} + +impl DocLayout { + /// The characters drawn as glyph id 0 on `pages` (0-based, inclusive; `None` is every page), + /// from the display list the PDF and PNG exports draw with `opts`: body, headers, footers, + /// text boxes, notes, table cells, charts, diagrams and equations. A character counts once per + /// glyph drawn for it. Default-ignorable and control characters never count (they draw + /// nothing), and a Private Use Area character counts only if it really drew as glyph 0. + /// Pages past the end are ignored. + pub fn missing_glyphs(&self, doc: &Document, opts: &DisplayOptions, pages: Option>) -> MissingGlyphs { + let mut found = Found::default(); + for (i, page) in self.pages.iter().enumerate() { + if pages.as_ref().is_some_and(|r| !r.contains(&i)) { + continue; + } + for d in page_display(doc, page, opts) { + found.walk(&d); + } + } + MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated } + } +} + +#[derive(Default)] +struct Found { + counts: BTreeMap, + truncated: bool, +} + +impl Found { + fn walk(&mut self, d: &Draw) { + match d { + Draw::Glyphs { face, glyphs, text, ranges, .. } => { + let one_each = ranges.is_empty() && text.chars().count() == glyphs.len(); + let mut chars = text.chars(); + for (k, (gid, ..)) in glyphs.iter().enumerate() { + // A run without ranges (a list label) maps one character to one glyph, when + // the counts agree; else it can't be traced and isn't counted. + let own = chars.next(); + if *gid != 0 { + continue; + } + let shown: Vec = if one_each { + own.into_iter().collect() + } else { + ranges.get(k).and_then(|r| text.get(r.clone())).map(|t| t.chars().collect()).unwrap_or_default() + }; + let drawn: Vec = shown.into_iter().filter(|c| draws_something(*c)).collect(); + // A cluster of several characters drew one empty box: it is the one the face + // lacks, not the letter beside it. + let lacking: Vec = drawn.iter().copied().filter(|c| !face.covers(*c)).collect(); + for c in if lacking.is_empty() { drawn } else { lacking } { + self.count(c); + } + } + } + Draw::Figure { draws: items, .. } | Draw::Turned { items, .. } | Draw::Rotated { items, .. } | Draw::Clip { items, .. } => { + for d in items { + self.walk(d); + } + } + _ => {} + } + } + + fn count(&mut self, c: char) { + if let Some(n) = self.counts.get_mut(&c) { + *n = n.saturating_add(1); + } else if self.counts.len() < MAX_MISSING_GLYPHS { + self.counts.insert(c, 1); + } else { + self.truncated = true; + } + } +} + +/// The byte range of `text` each of `shaped` shows: from its cluster to the next cluster start in +/// the text (or its end), whichever order the glyphs are in. A ligature's glyph covers all its +/// characters; the glyphs of one cluster share its range. Ranges always lie on character +/// boundaries of `text`. +pub(crate) fn cluster_ranges(text: &str, shaped: &[ShapedGlyph]) -> Vec> { + let mut starts: Vec = shaped.iter().map(|g| g.cluster.min(text.len())).filter(|b| text.is_char_boundary(*b)).collect(); + starts.sort_unstable(); + starts.dedup(); + shaped + .iter() + .map(|g| { + let from = g.cluster.min(text.len()); + let from = if text.is_char_boundary(from) { from } else { (0..from).rev().find(|b| text.is_char_boundary(*b)).unwrap_or(0) }; + let to = match starts.binary_search(&from) { + Ok(i) => starts.get(i + 1).copied().unwrap_or(text.len()), + Err(_) => text.len(), + }; + from..to.max(from) + }) + .collect() +} diff --git a/crates/layout/src/tests.rs b/crates/layout/src/tests.rs index fa7e4a111a..9ced469510 100644 --- a/crates/layout/src/tests.rs +++ b/crates/layout/src/tests.rs @@ -13,6 +13,7 @@ mod float_top; mod grid; mod ligature; pub(crate) mod linear; +mod missing_glyphs; mod nbsp; mod nested_cant_split; mod numeral; diff --git a/crates/layout/src/tests/missing_glyphs.rs b/crates/layout/src/tests/missing_glyphs.rs new file mode 100644 index 0000000000..944a459f2c --- /dev/null +++ b/crates/layout/src/tests/missing_glyphs.rs @@ -0,0 +1,252 @@ +//! [`DocLayout::missing_glyphs`] (strict export, #6005): glyph id 0 in the display list, named by +//! its character, wherever the text is drawn. + +use std::sync::Arc; + +use wordcraft_doc::graphic::{Graphic, GraphicItem, GraphicKind, TextAlign}; +use wordcraft_doc::para::InlineObject; +use wordcraft_doc::props::Rgb; +use wordcraft_doc::{Block, Document, Path, Pos, StoryRef, Table}; +use wordcraft_fonts::FontDb; + +use super::{footnote, lay, page_float, text_box_at}; +use crate::display::DisplayOptions; +use crate::{DocLayout, MAX_MISSING_GLYPHS, MissingGlyphs}; + +/// Code points in unassigned plane 4 that no face of the host covers, checked against the body +/// font and the whole fallback chain (so the test never guesses). +fn uncovered(n: usize) -> Vec { + let db = FontDb::global(); + let primary = wordcraft_fonts::word::resolve(&Document::default().settings.minor_font, false, false).face; + let out: Vec = (0x40000u32..0x4FFF0) + .filter_map(char::from_u32) + .filter(|c| { + !primary.covers(*c) && db.fallback_styled(*c, primary.id(), false, false).is_none() && db.fallback_for(*c, primary.id()).is_none() + }) + .take(n) + .collect(); + assert_eq!(out.len(), n, "the host has faces covering all of plane 4"); + out +} + +fn missing(d: &Document, l: &DocLayout) -> MissingGlyphs { + l.missing_glyphs(d, &DisplayOptions::default(), None) +} + +fn chars(m: &MissingGlyphs) -> Vec { + m.chars.iter().map(|(c, _)| *c).collect() +} + +fn hf_part(d: &mut Document, kind: wordcraft_doc::PartKind, text: &str) -> u32 { + d.add_part(kind, vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(text, Default::default()))]) +} + +fn chart_with(d: &mut Document, label: &str) { + let items = vec![GraphicItem::Text { + rect: [0.0, 100.0, 216.0, 20.0], + text: label.into(), + size: 10.0, + color: Rgb::BLACK, + bold: false, + align: TextAlign::Center, + font: None, + }]; + let graphic = Graphic { kind: GraphicKind::Chart, items, ..Default::default() }; + let obj = InlineObject::Graphic { w: 216.0, h: 144.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) }; + d.insert_object(&Pos::body(0, 0), obj, &Default::default()).unwrap(); +} + +fn equation_with(d: &mut Document, linear: &str) { + let math = wordcraft_doc::math::Math { nodes: wordcraft_doc::math::parse_linear(linear), ..Default::default() }; + let n = d.para_mut(StoryRef::Body, &Path::top(0)).unwrap().len(); + d.para_mut(StoryRef::Body, &Path::top(0)) + .unwrap() + .insert_object(n, InlineObject::Equation { linear: linear.into(), display: false, math }, &Default::default()) + .unwrap(); +} + +#[test] +fn nothing_is_missing_in_plain_text() { + let d = Document::from_text("Hello, world."); + assert!(missing(&d, &lay(&d)).is_empty()); +} + +#[test] +fn a_character_no_face_covers_is_counted_once_per_occurrence_in_the_body() { + let c = uncovered(1)[0]; + let d = Document::from_text(&format!("a{c}b{c}{c}c")); + let m = missing(&d, &lay(&d)); + assert_eq!(m, MissingGlyphs { chars: vec![(c, 3)], truncated: false }); +} + +#[test] +fn a_missing_glyph_is_found_in_a_header() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Body"); + d.last_section.headers.default = Some(hf_part(&mut d, wordcraft_doc::PartKind::Header, &format!("H{c}{c}"))); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 2)]); +} + +#[test] +fn a_missing_glyph_is_found_in_a_footer() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Body"); + d.last_section.footers.default = Some(hf_part(&mut d, wordcraft_doc::PartKind::Footer, &format!("F{c}"))); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]); +} + +#[test] +fn a_missing_glyph_is_found_in_a_footnote() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Body text."); + footnote(&mut d, &Pos::body(0, 4), &format!("Note {c}")); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]); +} + +#[test] +fn a_missing_glyph_is_found_in_a_text_box() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Body text."); + text_box_at(&mut d, &Pos::body(0, 0), &format!("Box {c}{c}{c}"), 120.0, 40.0, page_float(300.0, 300.0)); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 3)]); +} + +#[test] +fn a_missing_glyph_is_found_in_a_table_cell() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("before"); + let mut t = Table::new(1, 2, 300.0); + t.rows[0].cells[1].blocks = vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(&format!("cell {c}"), Default::default()))]; + d.insert_block(StoryRef::Body, &Path::top(1), Block::Table(t)).unwrap(); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 1)]); +} + +#[test] +fn a_missing_glyph_is_found_in_a_chart_label() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Chart"); + chart_with(&mut d, &format!("Q{c}Q{c}")); + assert_eq!(missing(&d, &lay(&d)).chars, vec![(c, 2)]); +} + +#[test] +fn a_chart_label_run_names_each_glyphs_characters() { + use crate::display::Draw; + let mut d = Document::from_text("Chart"); + chart_with(&mut d, "Q1 total"); + let l = lay(&d); + fn runs(d: &Draw, out: &mut Vec<(String, usize, usize)>) { + match d { + Draw::Glyphs { text, glyphs, ranges, .. } if text == "Q1 total" || !ranges.is_empty() => { + out.push((text.clone(), glyphs.len(), ranges.len())); + assert!(ranges.iter().all(|r| text.get(r.clone()).is_some()), "ranges on character boundaries"); + } + Draw::Figure { draws, .. } => draws.iter().for_each(|x| runs(x, out)), + _ => {} + } + } + let mut found = Vec::new(); + for x in crate::display::page_display(&d, &l.pages[0], &DisplayOptions::default()) { + runs(&x, &mut found); + } + assert!(found.iter().any(|(t, g, r)| t == "Q1 total" && g == r), "{found:?}"); +} + +#[test] +fn a_missing_glyph_is_found_in_an_equation() { + let c = uncovered(1)[0]; + let mut d = Document::from_text("Eq: "); + equation_with(&mut d, &format!("x+{c}")); + let m = missing(&d, &lay(&d)); + assert_eq!(m.chars, vec![(c, 1)], "{m:?}"); +} + +#[test] +fn names_in_many_scripts_report_nothing() { + if std::env::var_os("CRAFT_FONTS_DIR").is_none() { + eprintln!("CRAFT_FONTS_DIR unset: the multilingual faces aren't available, skipping"); + return; + } + let names = ["Łukasz Żmijewski", "Gheorghe Ștefănescu", "Krejčí", "Şahın Gül", "Llŷr Grŵn", "王小明", "김민수", "สมชาย ใจดี", "राम शर्मा"]; + let d = Document::from_text(&names.join("\n")); + let m = missing(&d, &lay(&d)); + assert!(m.is_empty(), "{m:?}"); +} + +#[test] +fn joiners_and_variation_selectors_are_never_counted() { + let d = Document::from_text("a\u{200D}b\u{200C}c\u{FE0F}d\u{FE0E}e\u{00AD}f\u{200E}g\u{2028}h\ti"); + assert!(missing(&d, &lay(&d)).is_empty()); + // A ZWJ emoji sequence: the joiners and selectors in it never appear, whatever the host's + // emoji face covers. + let d = Document::from_text("\u{1F469}\u{200D}\u{1F4BB} \u{2764}\u{FE0F} \u{1F44D}\u{1F3FD}"); + let m = missing(&d, &lay(&d)); + assert!(!chars(&m).iter().any(|c| matches!(c, '\u{200D}' | '\u{FE0F}' | '\u{FE0E}')), "{m:?}"); +} + +#[test] +fn a_selector_beside_a_missing_character_is_not_counted_with_it() { + let c = uncovered(1)[0]; + let d = Document::from_text(&format!("{c}\u{FE0F}")); + assert_eq!(chars(&missing(&d, &lay(&d))), vec![c]); +} + +#[test] +fn the_list_is_sorted_and_capped_at_256_distinct_code_points() { + let all = uncovered(MAX_MISSING_GLYPHS + 44); + let text: String = all.iter().rev().collect(); + let d = Document::from_text(&text); + let m = missing(&d, &lay(&d)); + assert!(m.truncated); + assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS); + assert!(m.chars.windows(2).all(|w| w[0].0 < w[1].0), "sorted by code point"); + // Exactly the cap's worth is no truncation. + let d = Document::from_text(&all.iter().take(MAX_MISSING_GLYPHS).collect::()); + let m = missing(&d, &lay(&d)); + assert!(!m.truncated); + assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS); +} + +#[test] +fn the_page_filter_selects_pages() { + let [x, y] = <[char; 2]>::try_from(uncovered(2)).unwrap(); + let d = Document::from_text(&format!("first {x}\n{}last {y}", "filler line\n".repeat(150))); + let l = lay(&d); + let last = l.pages.len() - 1; + assert!(last >= 1, "the document runs over pages"); + let on = |r| l.missing_glyphs(&d, &DisplayOptions::default(), r); + assert_eq!(on(None).chars, vec![(x.min(y), 1), (x.max(y), 1)]); + assert_eq!(on(Some(0..=0)).chars, vec![(x, 1)]); + assert_eq!(on(Some(last..=last)).chars, vec![(y, 1)]); + assert!(on(Some(last + 1..=last + 9)).is_empty()); + #[allow(clippy::reversed_empty_ranges)] + let backwards = Some(1..=0); + assert!(on(backwards).is_empty()); +} + +#[test] +fn an_equation_run_names_each_glyphs_characters() { + use crate::display::Draw; + let mut d = Document::from_text("Eq: "); + equation_with(&mut d, "x+ab"); + let l = lay(&d); + fn runs(d: &Draw, out: &mut Vec<(String, usize, usize)>) { + match d { + Draw::Glyphs { text, glyphs, ranges, .. } if !text.is_empty() => { + assert!(ranges.iter().all(|r| text.get(r.clone()).is_some()), "ranges on character boundaries"); + out.push((text.clone(), glyphs.len(), ranges.len())); + } + Draw::Figure { draws, .. } | Draw::Clip { items: draws, .. } | Draw::Rotated { items: draws, .. } => { + draws.iter().for_each(|x| runs(x, out)) + } + _ => {} + } + } + let mut found = Vec::new(); + for x in crate::display::page_display(&d, &l.pages[0], &DisplayOptions::default()) { + runs(&x, &mut found); + } + // The equation's runs (the body text "Eq: " is a run too) each carry one range per glyph. + assert!(found.iter().any(|(t, g, _)| t.contains('\u{1D44E}') && *g == 2), "{found:?}"); + assert!(found.iter().all(|(_, g, r)| g == r), "{found:?}"); +}