From 94a6466bf06bc9d00b6f886dbcf32cdece74b59a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 8 Nov 2026 22:54:37 +0000 Subject: [PATCH] #6005: machine reason codes, source_part and totals in the strict report JSON reasons are snake_case codes (missing_part, external_target, link_only, undecodable, unsupported_format); "in" becomes "source_part" (full story part); total_missing_glyphs and total_missing_pictures count past the 256 caps (code point bitset, saturating picture counter). Human lines unchanged. README documents the schema. Layout missing_glyphs tests call test_fonts(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8G9hbs6XYZFsgjbL --- README.md | 18 +++-- apps/wordcraft-cli/tests/it/strict.rs | 4 +- crates/doc/src/lib.rs | 2 +- crates/doc/src/missing.rs | 33 +++++++++ crates/engine/src/strict.rs | 49 +++++++----- crates/engine/tests/strict_export.rs | 35 ++++++++- crates/engine/tests/strict_pictures.rs | 3 +- crates/layout/src/missing.rs | 90 ++++++++++++++++++++++- crates/layout/src/strict.rs | 34 ++++++--- crates/layout/src/tests/missing_glyphs.rs | 20 ++--- 10 files changed, 230 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 169a81db92..27d718c0cd 100644 --- a/README.md +++ b/README.md @@ -174,19 +174,25 @@ output type, checked before the input is read) and 3 (strict refusal). On exit 3 ```sh wordcraft-cli convert in.docx out.pdf --strict # wordcraft-cli: strict: 1 character cannot be drawn: U+2D30 (×3) -# {"schema":1,"strict":"failed","missing_glyphs":[{"cp":"U+2D30","count":3}],"missing_pictures":[],"truncated":false} +# {"schema":1,"strict":"failed","missing_glyphs":[{"cp":"U+2D30","count":3}],"missing_pictures":[{"part":"word/media/image2.bin","source_part":"word/header2.xml","reason":"unsupported_format"}],"truncated":false,"total_missing_glyphs":1,"total_missing_pictures":1,"glyphs_truncated":false,"pictures_truncated":false} ``` -`missing_pictures` entries are `{"part", "in", "reason"}`; the reasons are `missing part`, -`external target`, `link only`, `undecodable` and `unsupported format`. Lists stop at 256 entries -and `truncated` says so. Only characters the output draws count (not hidden text, or deleted text +The JSON line (schema 1, also what `file.strictReport` returns) has `schema`, `strict` +(`"passed"` or `"failed"`), `missing_glyphs` (entries `{"cp", "count"}`, sorted by code point), +`missing_pictures` (entries `{"part", "source_part", "reason"}`: the picture's part or target, the +story part it sits in such as `word/header2.xml`, and a reason code), `truncated`, the two totals +`total_missing_glyphs` and `total_missing_pictures`, and `glyphs_truncated` and +`pictures_truncated` for each list. The reason codes are `missing_part`, `external_target`, +`link_only`, `undecodable` and `unsupported_format`; the stderr lines use plain words for them. +Each list stops at 256 entries separately; `truncated` is true when either was cut, and the totals +count every distinct character and picture found, those past the cap included. Only characters the output draws count (not hidden text, or deleted text without `--markup`), and a character a fallback font draws is fine. A Private Use Area character fails only if no installed font maps it. The result depends on the fonts installed on the host (see #5624 for a fixed font set). An external picture target is never fetched: it is reported, not loaded. Pictures dropped when a DOCX is opened (a missing part, an external target, or a `link only` picture with no embedded copy) are reported for DOCX input only. They have no page, so a PNG of any page is also refused for a missing or external picture anywhere in the document, unused headers included. `render --strict` checks only the page it writes (`--page`; `--scale` doesn't matter), prints nothing on stdout when it refuses, and reports the -same way. The engine takes `strict: true` on `file.save`, `file.exportPdf` and `file.exportPng`, -and `file.strictReport` returns the report without writing. +same way. The engine takes `strict: true` on `file.save`, `file.saveAs`, `file.exportPdf` and +`file.exportPng`, and `file.strictReport {markup?, page?}` returns the report without writing. Web: `cd apps/wordcraft-web && trunk serve`, then open . diff --git a/apps/wordcraft-cli/tests/it/strict.rs b/apps/wordcraft-cli/tests/it/strict.rs index 9bdc0c77b8..6695abb6d9 100644 --- a/apps/wordcraft-cli/tests/it/strict.rs +++ b/apps/wordcraft-cli/tests/it/strict.rs @@ -195,7 +195,7 @@ fn a_corrupt_header_picture_fails_strict_naming_the_part_and_the_header() { assert_eq!(json["strict"], "failed"); assert_eq!( json["missing_pictures"][0], - serde_json::json!({"part": "word/media/image2.bin", "in": "header2.xml", "reason": "unsupported format"}) + serde_json::json!({"part": "word/media/image2.bin", "source_part": "word/header2.xml", "reason": "unsupported_format"}) ); assert!(!out.exists()); eprintln!("{text}"); @@ -222,7 +222,7 @@ fn an_external_picture_target_is_never_fetched_and_is_reported_under_strict() { let strict = convert(&input, &out, true); assert_eq!(strict.status.code(), Some(3), "{}", stderr(&strict)); let json = report_json(&strict); - assert_eq!(json["missing_pictures"][0]["reason"], "external target"); + assert_eq!(json["missing_pictures"][0]["reason"], "external_target"); assert!(stderr(&strict).contains("external target")); assert!(!out.exists()); diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index 2066a6d986..e8b84b17eb 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -85,7 +85,7 @@ pub use block::{Block, Blocks, BoxBudget, MAX_TEXT_BOX_DEPTH}; pub use compare::para_block; use document::MediaIndex; pub use error::{DocError, Result}; -pub use missing::{MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, MissingPictures}; +pub use missing::{MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, MissingPictures, picture_hash}; pub use parts::{MAX_LINKED_BOXES, MAX_TEXT_BOX_INSET, NoteSeparators, Part, PartKind, PermRange, TEXT_BOX_INSET, TextBody, free_id}; pub use position::{Path, Pos, StoryRef}; pub use settings::{ diff --git a/crates/doc/src/missing.rs b/crates/doc/src/missing.rs index 939f8f4eb0..68e192d79d 100644 --- a/crates/doc/src/missing.rs +++ b/crates/doc/src/missing.rs @@ -37,6 +37,23 @@ impl MissingPictureReason { Self::UnsupportedFormat => "unsupported format", } } + + /// The reason a [`code`](Self::code) names. + pub fn from_code(code: &str) -> Option { + [Self::MissingPart, Self::ExternalTarget, Self::LinkOnly, Self::Undecodable, Self::UnsupportedFormat].into_iter().find(|r| r.code() == code) + } + + /// The stable snake_case code the JSON report carries (`"missing_part"`, `"external_target"`, + /// `"link_only"`, `"undecodable"`, `"unsupported_format"`); never reworded. + pub fn code(self) -> &'static str { + match self { + Self::MissingPart => "missing_part", + Self::ExternalTarget => "external_target", + Self::LinkOnly => "link_only", + Self::Undecodable => "undecodable", + Self::UnsupportedFormat => "unsupported_format", + } + } } /// One picture that is missing. @@ -57,6 +74,8 @@ pub struct MissingPictures { pub list: Vec, /// More than [`MAX_MISSING_PICTURES`] were found; the rest are not listed. pub truncated: bool, + /// Pictures found, those past the cap included (saturating). Each distinct one counts once. + pub total: u64, /// The main story's part name in the source file (`word/document.xml`). pub main: String, /// A header, footer, note or comment story's part id ([`crate::Document::parts`]) to the @@ -68,6 +87,8 @@ pub struct MissingPictures { pub media_parts: std::collections::BTreeMap, /// What `list` holds, for an O(1) test of "already there" (a hostile file has many). seen: std::collections::HashSet, + /// Hashes of the distinct pictures past the cap (8 bytes each), so the total counts them once. + unlisted: std::collections::HashSet, } impl MissingPictures { @@ -83,13 +104,25 @@ impl MissingPictures { } if self.list.len() >= MAX_MISSING_PICTURES { self.truncated = true; + if self.unlisted.insert(picture_hash(&item)) { + self.total = self.total.saturating_add(1); + } return; } + self.total = self.total.saturating_add(1); self.seen.insert(item.clone()); self.list.push(item); } } +/// A 64-bit hash of a record, to tell the ones past a cap apart without keeping them. +pub fn picture_hash(item: &MissingPicture) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + item.hash(&mut h); + h.finish() +} + impl PartialEq for MissingPictures { /// Not part of the document's content. fn eq(&self, _: &Self) -> bool { diff --git a/crates/engine/src/strict.rs b/crates/engine/src/strict.rs index 08203ad3a4..e6e8acec14 100644 --- a/crates/engine/src/strict.rs +++ b/crates/engine/src/strict.rs @@ -4,13 +4,12 @@ //! The check runs on the layout and display options the export itself uses, so what it reports is //! what the output lacks. -use std::collections::BTreeMap; use std::ops::RangeInclusive; use serde_json::{Value, json}; use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture}; use wordcraft_layout::display::DisplayOptions; -use wordcraft_layout::{DocLayout, LayoutCache, MAX_MISSING_GLYPHS, StrictOutput}; +use wordcraft_layout::{DocLayout, LayoutCache, StrictOutput}; use crate::io::ExportOptions; @@ -27,6 +26,10 @@ pub struct StrictReport { pub glyphs_truncated: bool, /// More pictures were missing than are listed. pub pictures_truncated: bool, + /// Distinct characters found missing, those past the list's cap included. + pub total_glyphs: usize, + /// Pictures found missing, those past the list's cap included. + pub total_pictures: u64, } impl StrictReport { @@ -46,8 +49,10 @@ impl StrictReport { "schema": SCHEMA, "strict": if self.passed() { "passed" } else { "failed" }, "missing_glyphs": self.glyphs.iter().map(|(c, n)| json!({"cp": code_point(*c), "count": n})).collect::>(), - "missing_pictures": self.pictures.iter().map(|p| json!({"part": p.part, "in": story_label(&p.story), "reason": p.reason.label()})).collect::>(), + "missing_pictures": self.pictures.iter().map(|p| json!({"part": p.part, "source_part": p.story, "reason": p.reason.code()})).collect::>(), "truncated": self.truncated(), + "total_missing_glyphs": self.total_glyphs, + "total_missing_pictures": self.total_pictures, "glyphs_truncated": self.glyphs_truncated, "pictures_truncated": self.pictures_truncated, }) @@ -73,7 +78,7 @@ impl StrictReport { } } -/// The human text of a report: (code point text, count) and (part, in, reason) as the JSON has them. +/// The human text of a report: (code point text, count) and (part, story label, reason words). fn human_parts(glyphs: &[(String, u64)], pictures: &[(String, String, String)], glyphs_cut: bool, pictures_cut: bool) -> Vec { let mut out = Vec::new(); if !glyphs.is_empty() { @@ -105,7 +110,7 @@ pub fn lines_from_json(report: &Value) -> Vec { let pictures: Vec<(String, String, String)> = report .get("missing_pictures") .and_then(Value::as_array) - .map(|a| a.iter().map(|p| (text(p, "part"), text(p, "in"), text(p, "reason"))).collect()) + .map(|a| a.iter().map(|p| (text(p, "part"), story_label(&text(p, "source_part")).to_string(), reason_label(&text(p, "reason")))).collect()) .unwrap_or_default(); let flag = |k: &str| report.get(k).and_then(Value::as_bool).unwrap_or(false); // A report without the per-list flags (the top-level one alone) marks both lists. @@ -116,6 +121,11 @@ pub fn lines_from_json(report: &Value) -> Vec { human_parts(&glyphs, &pictures, glyphs_cut, pictures_cut).into_iter().map(|p| format!("strict: {p}")).collect() } +/// The words for a JSON reason code; a code this version does not know is shown as it is. +fn reason_label(code: &str) -> String { + wordcraft_doc::MissingPictureReason::from_code(code).map_or_else(|| code.to_string(), |r| r.label().to_string()) +} + /// `U+2D30`: upper-case hex, at least four digits. fn code_point(c: char) -> String { format!("U+{:04X}", c as u32) @@ -143,25 +153,16 @@ pub fn strict_report( StrictOutput::Pdf => wordcraft_pdf::WATERMARK_MAX_CHARS, StrictOutput::Png => usize::MAX, }; - let wm_missing = wordcraft_layout::missing_watermark_glyphs(wm, max); - let mut all: BTreeMap = missing.chars.iter().copied().collect(); - missing.truncated |= wm_missing.truncated; - for (c, n) in wm_missing.chars { - if let Some(count) = all.get_mut(&c) { - *count = count.saturating_add(n); - } else if all.len() < MAX_MISSING_GLYPHS { - all.insert(c, n); - } else { - missing.truncated = true; - } - } - missing.chars = all.into_iter().collect(); + missing.merge(wordcraft_layout::missing_watermark_glyphs(wm, max)); } - let (undecodable, mut pictures_truncated) = wordcraft_layout::undecodable_pictures(doc, lay, pages, display, output); + let (undecodable, mut pictures_truncated, undecodable_total) = wordcraft_layout::undecodable_pictures(doc, lay, pages, display, output); pictures_truncated |= doc.missing_pictures.truncated; + // The two lists overlap where the same picture is in both; those are counted once. + let mut overlap: u64 = 0; let mut pictures: Vec = Vec::new(); for p in doc.missing_pictures.list.iter().cloned().chain(undecodable) { if pictures.contains(&p) { + overlap = overlap.saturating_add(1); continue; } if pictures.len() >= MAX_MISSING_PICTURES { @@ -170,7 +171,15 @@ pub fn strict_report( } pictures.push(p); } - StrictReport { glyphs: missing.chars, pictures, glyphs_truncated: missing.truncated, pictures_truncated } + let total_pictures = doc.missing_pictures.total.saturating_add(undecodable_total).saturating_sub(overlap).max(pictures.len() as u64); + StrictReport { + glyphs: missing.chars, + pictures, + glyphs_truncated: missing.truncated, + pictures_truncated, + total_glyphs: missing.total, + total_pictures, + } } /// The PDF export's options for `opts`: the one place they are built, shared by the export and diff --git a/crates/engine/tests/strict_export.rs b/crates/engine/tests/strict_export.rs index 0c4a67a210..995cc0c9ee 100644 --- a/crates/engine/tests/strict_export.rs +++ b/crates/engine/tests/strict_export.rs @@ -142,6 +142,8 @@ fn strict_report_has_the_schema_1_shape() { "missing_glyphs": [{"cp": format!("U+{:04X}", c as u32), "count": 2}], "missing_pictures": [], "truncated": false, + "total_missing_glyphs": 1, + "total_missing_pictures": 0, "glyphs_truncated": false, "pictures_truncated": false, }) @@ -154,7 +156,7 @@ fn strict_report_has_the_schema_1_shape() { let r = ok.run("file.strictReport", &json!({})).unwrap(); assert_eq!( r, - json!({"schema": 1, "strict": "passed", "missing_glyphs": [], "missing_pictures": [], "truncated": false, "glyphs_truncated": false, "pictures_truncated": false}) + json!({"schema": 1, "strict": "passed", "missing_glyphs": [], "missing_pictures": [], "truncated": false, "total_missing_glyphs": 0, "total_missing_pictures": 0, "glyphs_truncated": false, "pictures_truncated": false}) ); } @@ -181,7 +183,7 @@ fn an_undecodable_page_background_picture_is_refused_for_pdf_and_png() { let mut s = Session::new(doc); let r = s.run("file.strictReport", &json!({})).unwrap(); assert_eq!(r["strict"], "failed"); - assert_eq!(r["missing_pictures"][0], json!({"part": "bg.bin", "in": "background", "reason": "unsupported format"})); + assert_eq!(r["missing_pictures"][0], json!({"part": "bg.bin", "source_part": "background", "reason": "unsupported_format"})); let msg = refused(s.run("file.exportPdf", &json!({"path": d.join("a.pdf"), "strict": true}))); assert!(msg.contains("bg.bin (background)"), "{msg}"); refused(s.run("file.exportPng", &json!({"path": d.join("a.png"), "strict": true}))); @@ -268,6 +270,35 @@ fn each_list_says_for_itself_that_it_was_cut() { let lines = wordcraft_engine::strict::lines_from_json(&r); assert!(lines[0].ends_with(", …"), "{}", lines[0]); assert!(!lines[1].contains('…'), "{}", lines[1]); + assert_eq!((r["total_missing_glyphs"].clone(), r["total_missing_pictures"].clone()), (json!(300), json!(1))); +} + +#[test] +fn a_truncated_report_carries_totals_past_the_cap() { + let many = uncovered_many(300); + let mut doc = Document::from_text(&many.iter().collect::()); + for n in 0..300 { + doc.missing_pictures.push(wordcraft_doc::MissingPicture { + part: format!("word/media/gone{n}.png"), + story: "word/document.xml".into(), + reason: wordcraft_doc::MissingPictureReason::MissingPart, + }); + // The same record again is one picture. + doc.missing_pictures.push(wordcraft_doc::MissingPicture { + part: format!("word/media/gone{n}.png"), + story: "word/document.xml".into(), + reason: wordcraft_doc::MissingPictureReason::MissingPart, + }); + } + let mut s = Session::new(doc); + let r = s.run("file.strictReport", &json!({})).unwrap(); + assert_eq!(r["missing_glyphs"].as_array().unwrap().len(), 256); + assert_eq!(r["missing_pictures"].as_array().unwrap().len(), 256); + assert_eq!(r["truncated"], true); + assert_eq!(r["total_missing_glyphs"], 300); + assert_eq!(r["total_missing_pictures"], 300); + assert_eq!(r["missing_pictures"][0]["reason"], "missing_part"); + assert_eq!(r["missing_pictures"][0]["source_part"], "word/document.xml"); } #[test] diff --git a/crates/engine/tests/strict_pictures.rs b/crates/engine/tests/strict_pictures.rs index 63f2063d3d..5a3df44299 100644 --- a/crates/engine/tests/strict_pictures.rs +++ b/crates/engine/tests/strict_pictures.rs @@ -59,8 +59,9 @@ fn docx(header_media: &[u8], body_media: &[u8]) -> Vec { fn report(d: &Document, pages: Option>) -> Vec<(String, String, MissingPictureReason)> { let l = layout(d, &mut LayoutCache::new(), &LayoutOptions::default()); - let (list, truncated) = undecodable_pictures(d, &l, pages, &DisplayOptions::default(), StrictOutput::Pdf); + let (list, truncated, total) = undecodable_pictures(d, &l, pages, &DisplayOptions::default(), StrictOutput::Pdf); assert!(!truncated); + assert_eq!(total, list.len() as u64); list.into_iter().map(|m| (m.part, m.story, m.reason)).collect() } diff --git a/crates/layout/src/missing.rs b/crates/layout/src/missing.rs index e3b6795c9e..e1a6402d70 100644 --- a/crates/layout/src/missing.rs +++ b/crates/layout/src/missing.rs @@ -24,6 +24,11 @@ pub struct MissingGlyphs { pub chars: Vec<(char, u64)>, /// More than [`MAX_MISSING_GLYPHS`] distinct code points were missing: the rest aren't listed. pub truncated: bool, + /// How many distinct code points were missing, those past the cap included. + pub total: usize, + /// Every distinct code point found, kept only once the cap was passed (136 KiB), so merging + /// the watermark's can still count the distinct ones. + pub(crate) seen: Option>, } impl MissingGlyphs { @@ -31,6 +36,75 @@ impl MissingGlyphs { pub fn is_empty(&self) -> bool { self.chars.is_empty() && !self.truncated } + + /// A list of `chars` that fits under the cap: its total is its length. + pub fn from_listed(chars: Vec<(char, u64)>) -> Self { + let total = chars.len(); + Self { chars, truncated: false, total, seen: None } + } + + /// Add `other` (the watermark's characters): counts of a shared code point are summed, new + /// ones are listed while the cap allows, and the total counts each distinct one once. + pub fn merge(&mut self, other: MissingGlyphs) { + let mut all: BTreeMap = std::mem::take(&mut self.chars).into_iter().collect(); + let mut seen = self.seen.take(); + for (c, n) in other.chars { + if let Some(count) = all.get_mut(&c) { + *count = count.saturating_add(n); + } else if let Some(set) = seen.as_mut() { + set.insert(c); + } else if all.len() < MAX_MISSING_GLYPHS { + all.insert(c, n); + } else { + let mut set = CodePointSet::of(all.keys().copied()); + set.insert(c); + seen = Some(Box::new(set)); + } + } + if let Some(theirs) = other.seen { + seen.get_or_insert_with(|| Box::new(CodePointSet::of(all.keys().copied()))).union(&theirs); + } + self.truncated |= other.truncated || seen.is_some(); + self.total = seen.as_ref().map_or(all.len(), |s| s.len()); + self.chars = all.into_iter().collect(); + self.seen = seen; + } +} + +/// A set of code points as a bitset over the whole code point space: bounded at 136 KiB however +/// many distinct characters a hostile file holds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct CodePointSet(Vec); + +impl CodePointSet { + fn empty() -> Self { + Self(vec![0; 0x11_0000 / 64]) + } + + fn of(chars: impl Iterator) -> Self { + let mut set = Self::empty(); + for c in chars { + set.insert(c); + } + set + } + + fn insert(&mut self, c: char) { + let n = c as usize; + if let Some(word) = self.0.get_mut(n / 64) { + *word |= 1 << (n % 64); + } + } + + fn union(&mut self, other: &Self) { + for (a, b) in self.0.iter_mut().zip(&other.0) { + *a |= b; + } + } + + fn len(&self) -> usize { + self.0.iter().map(|w| w.count_ones() as usize).sum() + } } impl DocLayout { @@ -50,7 +124,7 @@ impl DocLayout { found.walk(&d); } } - MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated } + found.finish() } } @@ -78,16 +152,23 @@ pub fn missing_watermark_glyphs(wm: &wordcraft_doc::Watermark, max_chars: usize) let ranges = cluster_ranges(&text, &shaped); found.run(&face, shaped.iter().map(|g| g.gid), &text, &ranges); } - MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated } + found.finish() } #[derive(Default)] struct Found { counts: BTreeMap, truncated: bool, + /// Every distinct code point, started when the list passes its cap. + over: Option>, } impl Found { + fn finish(self) -> MissingGlyphs { + let total = self.over.as_ref().map_or(self.counts.len(), |s| s.len()); + MissingGlyphs { chars: self.counts.into_iter().collect(), truncated: self.truncated, total, seen: self.over } + } + fn walk(&mut self, d: &Draw) { match d { Draw::Glyphs { face, glyphs, text, ranges, .. } => self.run(face, glyphs.iter().map(|g| g.0), text, ranges), @@ -130,10 +211,15 @@ impl Found { fn count(&mut self, c: char) { if let Some(n) = self.counts.get_mut(&c) { *n = n.saturating_add(1); + } else if let Some(set) = self.over.as_mut() { + set.insert(c); } else if self.counts.len() < MAX_MISSING_GLYPHS { self.counts.insert(c, 1); } else { self.truncated = true; + let mut set = CodePointSet::of(self.counts.keys().copied()); + set.insert(c); + self.over = Some(Box::new(set)); } } } diff --git a/crates/layout/src/strict.rs b/crates/layout/src/strict.rs index 93a2548e1c..cac204b077 100644 --- a/crates/layout/src/strict.rs +++ b/crates/layout/src/strict.rs @@ -4,10 +4,10 @@ //! way (`wordcraft_geom::picture::decode_for_drawing`, and the metafile parser): what fails here //! is what shows as a grey box there. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ops::RangeInclusive; -use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, StoryRef}; +use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, StoryRef, picture_hash}; use wordcraft_geom::picture::DrawFailure; use crate::display::{DisplayOptions, Draw, ItemDisplay}; @@ -48,7 +48,8 @@ pub fn media_check(bytes: &[u8], output: StrictOutput) -> Result<(), DrawFailure /// The pictures drawn on `pages` (0-based, `None` for every page) that fail [`media_check`] for /// `output`, the page's background picture included (named `background`), one -/// per picture and story, at most [`MAX_MISSING_PICTURES`]; the flag says more were found. +/// per picture and story, at most [`MAX_MISSING_PICTURES`]; the flag says more were found, and the +/// count is how many distinct ones there were in all (saturating). /// /// `part` is the part the bytes were read from when the document knows it (a DOCX opened by the /// reader), else the media key. `story` is the source part of the header, footer, note or comment @@ -62,9 +63,12 @@ pub fn undecodable_pictures( pages: Option>, opts: &DisplayOptions, output: StrictOutput, -) -> (Vec, bool) { +) -> (Vec, bool, u64) { let mut out: Vec = Vec::new(); let mut truncated = false; + let mut total: u64 = 0; + // Hashes of the distinct ones past the cap, so the total counts each once. + let mut unlisted: HashSet = HashSet::new(); let mut verdicts: HashMap> = HashMap::new(); let range = pages.unwrap_or(0..=usize::MAX); for (n, page) in layout.pages.iter().enumerate() { @@ -96,8 +100,12 @@ pub fn undecodable_pictures( } if out.len() >= MAX_MISSING_PICTURES { truncated = true; - return (out, truncated); + if unlisted.insert(picture_hash(&item)) { + total = total.saturating_add(1); + } + continue; } + total = total.saturating_add(1); out.push(item); } } @@ -118,14 +126,20 @@ pub fn undecodable_pictures( }; let part = doc.missing_pictures.media_parts.get(&pic.media).cloned().unwrap_or_else(|| pic.media.clone()); let item = MissingPicture { part, story: "background".into(), reason }; - if out.len() >= MAX_MISSING_PICTURES { - truncated = true; - } else if !out.contains(&item) { - out.push(item); + if !out.contains(&item) { + if out.len() >= MAX_MISSING_PICTURES { + truncated = true; + if unlisted.insert(picture_hash(&item)) { + total = total.saturating_add(1); + } + } else { + total = total.saturating_add(1); + out.push(item); + } } } } - (out, truncated) + (out, truncated, total) } #[derive(Clone, Copy)] diff --git a/crates/layout/src/tests/missing_glyphs.rs b/crates/layout/src/tests/missing_glyphs.rs index 944a459f2c..1a9444f346 100644 --- a/crates/layout/src/tests/missing_glyphs.rs +++ b/crates/layout/src/tests/missing_glyphs.rs @@ -16,6 +16,7 @@ use crate::{DocLayout, MAX_MISSING_GLYPHS, MissingGlyphs}; /// Code points in unassigned plane 4 that no face of the host covers, checked against the body /// font and the whole fallback chain (so the test never guesses). fn uncovered(n: usize) -> Vec { + crate::test_fonts(); let db = FontDb::global(); let primary = wordcraft_fonts::word::resolve(&Document::default().settings.minor_font, false, false).face; let out: Vec = (0x40000u32..0x4FFF0) @@ -76,7 +77,7 @@ fn a_character_no_face_covers_is_counted_once_per_occurrence_in_the_body() { let c = uncovered(1)[0]; let d = Document::from_text(&format!("a{c}b{c}{c}c")); let m = missing(&d, &lay(&d)); - assert_eq!(m, MissingGlyphs { chars: vec![(c, 3)], truncated: false }); + assert_eq!(m, MissingGlyphs::from_listed(vec![(c, 3)])); } #[test] @@ -161,18 +162,6 @@ fn a_missing_glyph_is_found_in_an_equation() { assert_eq!(m.chars, vec![(c, 1)], "{m:?}"); } -#[test] -fn names_in_many_scripts_report_nothing() { - if std::env::var_os("CRAFT_FONTS_DIR").is_none() { - eprintln!("CRAFT_FONTS_DIR unset: the multilingual faces aren't available, skipping"); - return; - } - let names = ["Łukasz Żmijewski", "Gheorghe Ștefănescu", "Krejčí", "Şahın Gül", "Llŷr Grŵn", "王小明", "김민수", "สมชาย ใจดี", "राम शर्मा"]; - let d = Document::from_text(&names.join("\n")); - let m = missing(&d, &lay(&d)); - assert!(m.is_empty(), "{m:?}"); -} - #[test] fn joiners_and_variation_selectors_are_never_counted() { let d = Document::from_text("a\u{200D}b\u{200C}c\u{FE0F}d\u{FE0E}e\u{00AD}f\u{200E}g\u{2028}h\ti"); @@ -199,12 +188,14 @@ fn the_list_is_sorted_and_capped_at_256_distinct_code_points() { let m = missing(&d, &lay(&d)); assert!(m.truncated); assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS); + assert_eq!(m.total, MAX_MISSING_GLYPHS + 44, "the total counts the unlisted ones too"); assert!(m.chars.windows(2).all(|w| w[0].0 < w[1].0), "sorted by code point"); // Exactly the cap's worth is no truncation. let d = Document::from_text(&all.iter().take(MAX_MISSING_GLYPHS).collect::()); let m = missing(&d, &lay(&d)); assert!(!m.truncated); assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS); + assert_eq!(m.total, MAX_MISSING_GLYPHS); } #[test] @@ -247,6 +238,7 @@ fn an_equation_run_names_each_glyphs_characters() { runs(&x, &mut found); } // The equation's runs (the body text "Eq: " is a run too) each carry one range per glyph. - assert!(found.iter().any(|(t, g, _)| t.contains('\u{1D44E}') && *g == 2), "{found:?}"); + // The letters are the math italic ones where the font has them, else plain. + assert!(found.iter().any(|(t, g, _)| (t.contains('\u{1D44E}') || t == "ab") && *g == 2), "{found:?}"); assert!(found.iter().all(|(_, g, r)| g == r), "{found:?}"); }