mirror of
https://github.com/storytold/wordcraft.git
synced 2026-11-10 12:20:12 -06:00
#6005: machine reason codes, source_part and totals in the strict report
JSON reasons are snake_case codes (missing_part, external_target, link_only, undecodable, unsupported_format); "in" becomes "source_part" (full story part); total_missing_glyphs and total_missing_pictures count past the 256 caps (code point bitset, saturating picture counter). Human lines unchanged. README documents the schema. Layout missing_glyphs tests call test_fonts(). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8G9hbs6XYZFsgjbL
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
44e188d8c0
commit
94a6466bf0
@@ -174,19 +174,25 @@ output type, checked before the input is read) and 3 (strict refusal). On exit 3
|
||||
```sh
|
||||
wordcraft-cli convert in.docx out.pdf --strict
|
||||
# wordcraft-cli: strict: 1 character cannot be drawn: U+2D30 (×3)
|
||||
# {"schema":1,"strict":"failed","missing_glyphs":[{"cp":"U+2D30","count":3}],"missing_pictures":[],"truncated":false}
|
||||
# {"schema":1,"strict":"failed","missing_glyphs":[{"cp":"U+2D30","count":3}],"missing_pictures":[{"part":"word/media/image2.bin","source_part":"word/header2.xml","reason":"unsupported_format"}],"truncated":false,"total_missing_glyphs":1,"total_missing_pictures":1,"glyphs_truncated":false,"pictures_truncated":false}
|
||||
```
|
||||
|
||||
`missing_pictures` entries are `{"part", "in", "reason"}`; the reasons are `missing part`,
|
||||
`external target`, `link only`, `undecodable` and `unsupported format`. Lists stop at 256 entries
|
||||
and `truncated` says so. Only characters the output draws count (not hidden text, or deleted text
|
||||
The JSON line (schema 1, also what `file.strictReport` returns) has `schema`, `strict`
|
||||
(`"passed"` or `"failed"`), `missing_glyphs` (entries `{"cp", "count"}`, sorted by code point),
|
||||
`missing_pictures` (entries `{"part", "source_part", "reason"}`: the picture's part or target, the
|
||||
story part it sits in such as `word/header2.xml`, and a reason code), `truncated`, the two totals
|
||||
`total_missing_glyphs` and `total_missing_pictures`, and `glyphs_truncated` and
|
||||
`pictures_truncated` for each list. The reason codes are `missing_part`, `external_target`,
|
||||
`link_only`, `undecodable` and `unsupported_format`; the stderr lines use plain words for them.
|
||||
Each list stops at 256 entries separately; `truncated` is true when either was cut, and the totals
|
||||
count every distinct character and picture found, those past the cap included. Only characters the output draws count (not hidden text, or deleted text
|
||||
without `--markup`), and a character a fallback font draws is fine. A Private Use Area character
|
||||
fails only if no installed font maps it. The result depends on the fonts installed on the host
|
||||
(see #5624 for a fixed font set). An external picture target is never fetched: it is reported, not
|
||||
loaded. Pictures dropped when a DOCX is opened (a missing part, an external target, or a `link only` picture with no embedded copy) are reported for DOCX input only. They have no page, so a PNG of any page is also refused for a missing or external picture anywhere in the document, unused headers included. `render --strict` checks only the page it
|
||||
writes (`--page`; `--scale` doesn't matter), prints nothing on stdout when it refuses, and reports the
|
||||
same way. The engine takes `strict: true` on `file.save`, `file.exportPdf` and `file.exportPng`,
|
||||
and `file.strictReport` returns the report without writing.
|
||||
same way. The engine takes `strict: true` on `file.save`, `file.saveAs`, `file.exportPdf` and
|
||||
`file.exportPng`, and `file.strictReport {markup?, page?}` returns the report without writing.
|
||||
|
||||
Web: `cd apps/wordcraft-web && trunk serve`, then open <http://127.0.0.1:8771/?sample>.
|
||||
|
||||
|
||||
@@ -195,7 +195,7 @@ fn a_corrupt_header_picture_fails_strict_naming_the_part_and_the_header() {
|
||||
assert_eq!(json["strict"], "failed");
|
||||
assert_eq!(
|
||||
json["missing_pictures"][0],
|
||||
serde_json::json!({"part": "word/media/image2.bin", "in": "header2.xml", "reason": "unsupported format"})
|
||||
serde_json::json!({"part": "word/media/image2.bin", "source_part": "word/header2.xml", "reason": "unsupported_format"})
|
||||
);
|
||||
assert!(!out.exists());
|
||||
eprintln!("{text}");
|
||||
@@ -222,7 +222,7 @@ fn an_external_picture_target_is_never_fetched_and_is_reported_under_strict() {
|
||||
let strict = convert(&input, &out, true);
|
||||
assert_eq!(strict.status.code(), Some(3), "{}", stderr(&strict));
|
||||
let json = report_json(&strict);
|
||||
assert_eq!(json["missing_pictures"][0]["reason"], "external target");
|
||||
assert_eq!(json["missing_pictures"][0]["reason"], "external_target");
|
||||
assert!(stderr(&strict).contains("external target"));
|
||||
assert!(!out.exists());
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ pub use block::{Block, Blocks, BoxBudget, MAX_TEXT_BOX_DEPTH};
|
||||
pub use compare::para_block;
|
||||
use document::MediaIndex;
|
||||
pub use error::{DocError, Result};
|
||||
pub use missing::{MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, MissingPictures};
|
||||
pub use missing::{MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, MissingPictures, picture_hash};
|
||||
pub use parts::{MAX_LINKED_BOXES, MAX_TEXT_BOX_INSET, NoteSeparators, Part, PartKind, PermRange, TEXT_BOX_INSET, TextBody, free_id};
|
||||
pub use position::{Path, Pos, StoryRef};
|
||||
pub use settings::{
|
||||
|
||||
@@ -37,6 +37,23 @@ impl MissingPictureReason {
|
||||
Self::UnsupportedFormat => "unsupported format",
|
||||
}
|
||||
}
|
||||
|
||||
/// The reason a [`code`](Self::code) names.
|
||||
pub fn from_code(code: &str) -> Option<Self> {
|
||||
[Self::MissingPart, Self::ExternalTarget, Self::LinkOnly, Self::Undecodable, Self::UnsupportedFormat].into_iter().find(|r| r.code() == code)
|
||||
}
|
||||
|
||||
/// The stable snake_case code the JSON report carries (`"missing_part"`, `"external_target"`,
|
||||
/// `"link_only"`, `"undecodable"`, `"unsupported_format"`); never reworded.
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
Self::MissingPart => "missing_part",
|
||||
Self::ExternalTarget => "external_target",
|
||||
Self::LinkOnly => "link_only",
|
||||
Self::Undecodable => "undecodable",
|
||||
Self::UnsupportedFormat => "unsupported_format",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One picture that is missing.
|
||||
@@ -57,6 +74,8 @@ pub struct MissingPictures {
|
||||
pub list: Vec<MissingPicture>,
|
||||
/// More than [`MAX_MISSING_PICTURES`] were found; the rest are not listed.
|
||||
pub truncated: bool,
|
||||
/// Pictures found, those past the cap included (saturating). Each distinct one counts once.
|
||||
pub total: u64,
|
||||
/// The main story's part name in the source file (`word/document.xml`).
|
||||
pub main: String,
|
||||
/// A header, footer, note or comment story's part id ([`crate::Document::parts`]) to the
|
||||
@@ -68,6 +87,8 @@ pub struct MissingPictures {
|
||||
pub media_parts: std::collections::BTreeMap<String, String>,
|
||||
/// What `list` holds, for an O(1) test of "already there" (a hostile file has many).
|
||||
seen: std::collections::HashSet<MissingPicture>,
|
||||
/// Hashes of the distinct pictures past the cap (8 bytes each), so the total counts them once.
|
||||
unlisted: std::collections::HashSet<u64>,
|
||||
}
|
||||
|
||||
impl MissingPictures {
|
||||
@@ -83,13 +104,25 @@ impl MissingPictures {
|
||||
}
|
||||
if self.list.len() >= MAX_MISSING_PICTURES {
|
||||
self.truncated = true;
|
||||
if self.unlisted.insert(picture_hash(&item)) {
|
||||
self.total = self.total.saturating_add(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
self.total = self.total.saturating_add(1);
|
||||
self.seen.insert(item.clone());
|
||||
self.list.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
/// A 64-bit hash of a record, to tell the ones past a cap apart without keeping them.
|
||||
pub fn picture_hash(item: &MissingPicture) -> u64 {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
item.hash(&mut h);
|
||||
h.finish()
|
||||
}
|
||||
|
||||
impl PartialEq for MissingPictures {
|
||||
/// Not part of the document's content.
|
||||
fn eq(&self, _: &Self) -> bool {
|
||||
|
||||
+29
-20
@@ -4,13 +4,12 @@
|
||||
//! The check runs on the layout and display options the export itself uses, so what it reports is
|
||||
//! what the output lacks.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::ops::RangeInclusive;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture};
|
||||
use wordcraft_layout::display::DisplayOptions;
|
||||
use wordcraft_layout::{DocLayout, LayoutCache, MAX_MISSING_GLYPHS, StrictOutput};
|
||||
use wordcraft_layout::{DocLayout, LayoutCache, StrictOutput};
|
||||
|
||||
use crate::io::ExportOptions;
|
||||
|
||||
@@ -27,6 +26,10 @@ pub struct StrictReport {
|
||||
pub glyphs_truncated: bool,
|
||||
/// More pictures were missing than are listed.
|
||||
pub pictures_truncated: bool,
|
||||
/// Distinct characters found missing, those past the list's cap included.
|
||||
pub total_glyphs: usize,
|
||||
/// Pictures found missing, those past the list's cap included.
|
||||
pub total_pictures: u64,
|
||||
}
|
||||
|
||||
impl StrictReport {
|
||||
@@ -46,8 +49,10 @@ impl StrictReport {
|
||||
"schema": SCHEMA,
|
||||
"strict": if self.passed() { "passed" } else { "failed" },
|
||||
"missing_glyphs": self.glyphs.iter().map(|(c, n)| json!({"cp": code_point(*c), "count": n})).collect::<Vec<_>>(),
|
||||
"missing_pictures": self.pictures.iter().map(|p| json!({"part": p.part, "in": story_label(&p.story), "reason": p.reason.label()})).collect::<Vec<_>>(),
|
||||
"missing_pictures": self.pictures.iter().map(|p| json!({"part": p.part, "source_part": p.story, "reason": p.reason.code()})).collect::<Vec<_>>(),
|
||||
"truncated": self.truncated(),
|
||||
"total_missing_glyphs": self.total_glyphs,
|
||||
"total_missing_pictures": self.total_pictures,
|
||||
"glyphs_truncated": self.glyphs_truncated,
|
||||
"pictures_truncated": self.pictures_truncated,
|
||||
})
|
||||
@@ -73,7 +78,7 @@ impl StrictReport {
|
||||
}
|
||||
}
|
||||
|
||||
/// The human text of a report: (code point text, count) and (part, in, reason) as the JSON has them.
|
||||
/// The human text of a report: (code point text, count) and (part, story label, reason words).
|
||||
fn human_parts(glyphs: &[(String, u64)], pictures: &[(String, String, String)], glyphs_cut: bool, pictures_cut: bool) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if !glyphs.is_empty() {
|
||||
@@ -105,7 +110,7 @@ pub fn lines_from_json(report: &Value) -> Vec<String> {
|
||||
let pictures: Vec<(String, String, String)> = report
|
||||
.get("missing_pictures")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().map(|p| (text(p, "part"), text(p, "in"), text(p, "reason"))).collect())
|
||||
.map(|a| a.iter().map(|p| (text(p, "part"), story_label(&text(p, "source_part")).to_string(), reason_label(&text(p, "reason")))).collect())
|
||||
.unwrap_or_default();
|
||||
let flag = |k: &str| report.get(k).and_then(Value::as_bool).unwrap_or(false);
|
||||
// A report without the per-list flags (the top-level one alone) marks both lists.
|
||||
@@ -116,6 +121,11 @@ pub fn lines_from_json(report: &Value) -> Vec<String> {
|
||||
human_parts(&glyphs, &pictures, glyphs_cut, pictures_cut).into_iter().map(|p| format!("strict: {p}")).collect()
|
||||
}
|
||||
|
||||
/// The words for a JSON reason code; a code this version does not know is shown as it is.
|
||||
fn reason_label(code: &str) -> String {
|
||||
wordcraft_doc::MissingPictureReason::from_code(code).map_or_else(|| code.to_string(), |r| r.label().to_string())
|
||||
}
|
||||
|
||||
/// `U+2D30`: upper-case hex, at least four digits.
|
||||
fn code_point(c: char) -> String {
|
||||
format!("U+{:04X}", c as u32)
|
||||
@@ -143,25 +153,16 @@ pub fn strict_report(
|
||||
StrictOutput::Pdf => wordcraft_pdf::WATERMARK_MAX_CHARS,
|
||||
StrictOutput::Png => usize::MAX,
|
||||
};
|
||||
let wm_missing = wordcraft_layout::missing_watermark_glyphs(wm, max);
|
||||
let mut all: BTreeMap<char, u64> = missing.chars.iter().copied().collect();
|
||||
missing.truncated |= wm_missing.truncated;
|
||||
for (c, n) in wm_missing.chars {
|
||||
if let Some(count) = all.get_mut(&c) {
|
||||
*count = count.saturating_add(n);
|
||||
} else if all.len() < MAX_MISSING_GLYPHS {
|
||||
all.insert(c, n);
|
||||
} else {
|
||||
missing.truncated = true;
|
||||
}
|
||||
}
|
||||
missing.chars = all.into_iter().collect();
|
||||
missing.merge(wordcraft_layout::missing_watermark_glyphs(wm, max));
|
||||
}
|
||||
let (undecodable, mut pictures_truncated) = wordcraft_layout::undecodable_pictures(doc, lay, pages, display, output);
|
||||
let (undecodable, mut pictures_truncated, undecodable_total) = wordcraft_layout::undecodable_pictures(doc, lay, pages, display, output);
|
||||
pictures_truncated |= doc.missing_pictures.truncated;
|
||||
// The two lists overlap where the same picture is in both; those are counted once.
|
||||
let mut overlap: u64 = 0;
|
||||
let mut pictures: Vec<MissingPicture> = Vec::new();
|
||||
for p in doc.missing_pictures.list.iter().cloned().chain(undecodable) {
|
||||
if pictures.contains(&p) {
|
||||
overlap = overlap.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
if pictures.len() >= MAX_MISSING_PICTURES {
|
||||
@@ -170,7 +171,15 @@ pub fn strict_report(
|
||||
}
|
||||
pictures.push(p);
|
||||
}
|
||||
StrictReport { glyphs: missing.chars, pictures, glyphs_truncated: missing.truncated, pictures_truncated }
|
||||
let total_pictures = doc.missing_pictures.total.saturating_add(undecodable_total).saturating_sub(overlap).max(pictures.len() as u64);
|
||||
StrictReport {
|
||||
glyphs: missing.chars,
|
||||
pictures,
|
||||
glyphs_truncated: missing.truncated,
|
||||
pictures_truncated,
|
||||
total_glyphs: missing.total,
|
||||
total_pictures,
|
||||
}
|
||||
}
|
||||
|
||||
/// The PDF export's options for `opts`: the one place they are built, shared by the export and
|
||||
|
||||
@@ -142,6 +142,8 @@ fn strict_report_has_the_schema_1_shape() {
|
||||
"missing_glyphs": [{"cp": format!("U+{:04X}", c as u32), "count": 2}],
|
||||
"missing_pictures": [],
|
||||
"truncated": false,
|
||||
"total_missing_glyphs": 1,
|
||||
"total_missing_pictures": 0,
|
||||
"glyphs_truncated": false,
|
||||
"pictures_truncated": false,
|
||||
})
|
||||
@@ -154,7 +156,7 @@ fn strict_report_has_the_schema_1_shape() {
|
||||
let r = ok.run("file.strictReport", &json!({})).unwrap();
|
||||
assert_eq!(
|
||||
r,
|
||||
json!({"schema": 1, "strict": "passed", "missing_glyphs": [], "missing_pictures": [], "truncated": false, "glyphs_truncated": false, "pictures_truncated": false})
|
||||
json!({"schema": 1, "strict": "passed", "missing_glyphs": [], "missing_pictures": [], "truncated": false, "total_missing_glyphs": 0, "total_missing_pictures": 0, "glyphs_truncated": false, "pictures_truncated": false})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -181,7 +183,7 @@ fn an_undecodable_page_background_picture_is_refused_for_pdf_and_png() {
|
||||
let mut s = Session::new(doc);
|
||||
let r = s.run("file.strictReport", &json!({})).unwrap();
|
||||
assert_eq!(r["strict"], "failed");
|
||||
assert_eq!(r["missing_pictures"][0], json!({"part": "bg.bin", "in": "background", "reason": "unsupported format"}));
|
||||
assert_eq!(r["missing_pictures"][0], json!({"part": "bg.bin", "source_part": "background", "reason": "unsupported_format"}));
|
||||
let msg = refused(s.run("file.exportPdf", &json!({"path": d.join("a.pdf"), "strict": true})));
|
||||
assert!(msg.contains("bg.bin (background)"), "{msg}");
|
||||
refused(s.run("file.exportPng", &json!({"path": d.join("a.png"), "strict": true})));
|
||||
@@ -268,6 +270,35 @@ fn each_list_says_for_itself_that_it_was_cut() {
|
||||
let lines = wordcraft_engine::strict::lines_from_json(&r);
|
||||
assert!(lines[0].ends_with(", …"), "{}", lines[0]);
|
||||
assert!(!lines[1].contains('…'), "{}", lines[1]);
|
||||
assert_eq!((r["total_missing_glyphs"].clone(), r["total_missing_pictures"].clone()), (json!(300), json!(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_truncated_report_carries_totals_past_the_cap() {
|
||||
let many = uncovered_many(300);
|
||||
let mut doc = Document::from_text(&many.iter().collect::<String>());
|
||||
for n in 0..300 {
|
||||
doc.missing_pictures.push(wordcraft_doc::MissingPicture {
|
||||
part: format!("word/media/gone{n}.png"),
|
||||
story: "word/document.xml".into(),
|
||||
reason: wordcraft_doc::MissingPictureReason::MissingPart,
|
||||
});
|
||||
// The same record again is one picture.
|
||||
doc.missing_pictures.push(wordcraft_doc::MissingPicture {
|
||||
part: format!("word/media/gone{n}.png"),
|
||||
story: "word/document.xml".into(),
|
||||
reason: wordcraft_doc::MissingPictureReason::MissingPart,
|
||||
});
|
||||
}
|
||||
let mut s = Session::new(doc);
|
||||
let r = s.run("file.strictReport", &json!({})).unwrap();
|
||||
assert_eq!(r["missing_glyphs"].as_array().unwrap().len(), 256);
|
||||
assert_eq!(r["missing_pictures"].as_array().unwrap().len(), 256);
|
||||
assert_eq!(r["truncated"], true);
|
||||
assert_eq!(r["total_missing_glyphs"], 300);
|
||||
assert_eq!(r["total_missing_pictures"], 300);
|
||||
assert_eq!(r["missing_pictures"][0]["reason"], "missing_part");
|
||||
assert_eq!(r["missing_pictures"][0]["source_part"], "word/document.xml");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -59,8 +59,9 @@ fn docx(header_media: &[u8], body_media: &[u8]) -> Vec<u8> {
|
||||
|
||||
fn report(d: &Document, pages: Option<std::ops::RangeInclusive<usize>>) -> Vec<(String, String, MissingPictureReason)> {
|
||||
let l = layout(d, &mut LayoutCache::new(), &LayoutOptions::default());
|
||||
let (list, truncated) = undecodable_pictures(d, &l, pages, &DisplayOptions::default(), StrictOutput::Pdf);
|
||||
let (list, truncated, total) = undecodable_pictures(d, &l, pages, &DisplayOptions::default(), StrictOutput::Pdf);
|
||||
assert!(!truncated);
|
||||
assert_eq!(total, list.len() as u64);
|
||||
list.into_iter().map(|m| (m.part, m.story, m.reason)).collect()
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,11 @@ pub struct MissingGlyphs {
|
||||
pub chars: Vec<(char, u64)>,
|
||||
/// More than [`MAX_MISSING_GLYPHS`] distinct code points were missing: the rest aren't listed.
|
||||
pub truncated: bool,
|
||||
/// How many distinct code points were missing, those past the cap included.
|
||||
pub total: usize,
|
||||
/// Every distinct code point found, kept only once the cap was passed (136 KiB), so merging
|
||||
/// the watermark's can still count the distinct ones.
|
||||
pub(crate) seen: Option<Box<CodePointSet>>,
|
||||
}
|
||||
|
||||
impl MissingGlyphs {
|
||||
@@ -31,6 +36,75 @@ impl MissingGlyphs {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.chars.is_empty() && !self.truncated
|
||||
}
|
||||
|
||||
/// A list of `chars` that fits under the cap: its total is its length.
|
||||
pub fn from_listed(chars: Vec<(char, u64)>) -> Self {
|
||||
let total = chars.len();
|
||||
Self { chars, truncated: false, total, seen: None }
|
||||
}
|
||||
|
||||
/// Add `other` (the watermark's characters): counts of a shared code point are summed, new
|
||||
/// ones are listed while the cap allows, and the total counts each distinct one once.
|
||||
pub fn merge(&mut self, other: MissingGlyphs) {
|
||||
let mut all: BTreeMap<char, u64> = std::mem::take(&mut self.chars).into_iter().collect();
|
||||
let mut seen = self.seen.take();
|
||||
for (c, n) in other.chars {
|
||||
if let Some(count) = all.get_mut(&c) {
|
||||
*count = count.saturating_add(n);
|
||||
} else if let Some(set) = seen.as_mut() {
|
||||
set.insert(c);
|
||||
} else if all.len() < MAX_MISSING_GLYPHS {
|
||||
all.insert(c, n);
|
||||
} else {
|
||||
let mut set = CodePointSet::of(all.keys().copied());
|
||||
set.insert(c);
|
||||
seen = Some(Box::new(set));
|
||||
}
|
||||
}
|
||||
if let Some(theirs) = other.seen {
|
||||
seen.get_or_insert_with(|| Box::new(CodePointSet::of(all.keys().copied()))).union(&theirs);
|
||||
}
|
||||
self.truncated |= other.truncated || seen.is_some();
|
||||
self.total = seen.as_ref().map_or(all.len(), |s| s.len());
|
||||
self.chars = all.into_iter().collect();
|
||||
self.seen = seen;
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of code points as a bitset over the whole code point space: bounded at 136 KiB however
|
||||
/// many distinct characters a hostile file holds.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CodePointSet(Vec<u64>);
|
||||
|
||||
impl CodePointSet {
|
||||
fn empty() -> Self {
|
||||
Self(vec![0; 0x11_0000 / 64])
|
||||
}
|
||||
|
||||
fn of(chars: impl Iterator<Item = char>) -> Self {
|
||||
let mut set = Self::empty();
|
||||
for c in chars {
|
||||
set.insert(c);
|
||||
}
|
||||
set
|
||||
}
|
||||
|
||||
fn insert(&mut self, c: char) {
|
||||
let n = c as usize;
|
||||
if let Some(word) = self.0.get_mut(n / 64) {
|
||||
*word |= 1 << (n % 64);
|
||||
}
|
||||
}
|
||||
|
||||
fn union(&mut self, other: &Self) {
|
||||
for (a, b) in self.0.iter_mut().zip(&other.0) {
|
||||
*a |= b;
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.0.iter().map(|w| w.count_ones() as usize).sum()
|
||||
}
|
||||
}
|
||||
|
||||
impl DocLayout {
|
||||
@@ -50,7 +124,7 @@ impl DocLayout {
|
||||
found.walk(&d);
|
||||
}
|
||||
}
|
||||
MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated }
|
||||
found.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,16 +152,23 @@ pub fn missing_watermark_glyphs(wm: &wordcraft_doc::Watermark, max_chars: usize)
|
||||
let ranges = cluster_ranges(&text, &shaped);
|
||||
found.run(&face, shaped.iter().map(|g| g.gid), &text, &ranges);
|
||||
}
|
||||
MissingGlyphs { chars: found.counts.into_iter().collect(), truncated: found.truncated }
|
||||
found.finish()
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Found {
|
||||
counts: BTreeMap<char, u64>,
|
||||
truncated: bool,
|
||||
/// Every distinct code point, started when the list passes its cap.
|
||||
over: Option<Box<CodePointSet>>,
|
||||
}
|
||||
|
||||
impl Found {
|
||||
fn finish(self) -> MissingGlyphs {
|
||||
let total = self.over.as_ref().map_or(self.counts.len(), |s| s.len());
|
||||
MissingGlyphs { chars: self.counts.into_iter().collect(), truncated: self.truncated, total, seen: self.over }
|
||||
}
|
||||
|
||||
fn walk(&mut self, d: &Draw) {
|
||||
match d {
|
||||
Draw::Glyphs { face, glyphs, text, ranges, .. } => self.run(face, glyphs.iter().map(|g| g.0), text, ranges),
|
||||
@@ -130,10 +211,15 @@ impl Found {
|
||||
fn count(&mut self, c: char) {
|
||||
if let Some(n) = self.counts.get_mut(&c) {
|
||||
*n = n.saturating_add(1);
|
||||
} else if let Some(set) = self.over.as_mut() {
|
||||
set.insert(c);
|
||||
} else if self.counts.len() < MAX_MISSING_GLYPHS {
|
||||
self.counts.insert(c, 1);
|
||||
} else {
|
||||
self.truncated = true;
|
||||
let mut set = CodePointSet::of(self.counts.keys().copied());
|
||||
set.insert(c);
|
||||
self.over = Some(Box::new(set));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+24
-10
@@ -4,10 +4,10 @@
|
||||
//! way (`wordcraft_geom::picture::decode_for_drawing`, and the metafile parser): what fails here
|
||||
//! is what shows as a grey box there.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::ops::RangeInclusive;
|
||||
|
||||
use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, StoryRef};
|
||||
use wordcraft_doc::{Document, MAX_MISSING_PICTURES, MissingPicture, MissingPictureReason, StoryRef, picture_hash};
|
||||
use wordcraft_geom::picture::DrawFailure;
|
||||
|
||||
use crate::display::{DisplayOptions, Draw, ItemDisplay};
|
||||
@@ -48,7 +48,8 @@ pub fn media_check(bytes: &[u8], output: StrictOutput) -> Result<(), DrawFailure
|
||||
|
||||
/// The pictures drawn on `pages` (0-based, `None` for every page) that fail [`media_check`] for
|
||||
/// `output`, the page's background picture included (named `background`), one
|
||||
/// per picture and story, at most [`MAX_MISSING_PICTURES`]; the flag says more were found.
|
||||
/// per picture and story, at most [`MAX_MISSING_PICTURES`]; the flag says more were found, and the
|
||||
/// count is how many distinct ones there were in all (saturating).
|
||||
///
|
||||
/// `part` is the part the bytes were read from when the document knows it (a DOCX opened by the
|
||||
/// reader), else the media key. `story` is the source part of the header, footer, note or comment
|
||||
@@ -62,9 +63,12 @@ pub fn undecodable_pictures(
|
||||
pages: Option<RangeInclusive<usize>>,
|
||||
opts: &DisplayOptions,
|
||||
output: StrictOutput,
|
||||
) -> (Vec<MissingPicture>, bool) {
|
||||
) -> (Vec<MissingPicture>, bool, u64) {
|
||||
let mut out: Vec<MissingPicture> = Vec::new();
|
||||
let mut truncated = false;
|
||||
let mut total: u64 = 0;
|
||||
// Hashes of the distinct ones past the cap, so the total counts each once.
|
||||
let mut unlisted: HashSet<u64> = HashSet::new();
|
||||
let mut verdicts: HashMap<String, Option<DrawFailure>> = HashMap::new();
|
||||
let range = pages.unwrap_or(0..=usize::MAX);
|
||||
for (n, page) in layout.pages.iter().enumerate() {
|
||||
@@ -96,8 +100,12 @@ pub fn undecodable_pictures(
|
||||
}
|
||||
if out.len() >= MAX_MISSING_PICTURES {
|
||||
truncated = true;
|
||||
return (out, truncated);
|
||||
if unlisted.insert(picture_hash(&item)) {
|
||||
total = total.saturating_add(1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
total = total.saturating_add(1);
|
||||
out.push(item);
|
||||
}
|
||||
}
|
||||
@@ -118,14 +126,20 @@ pub fn undecodable_pictures(
|
||||
};
|
||||
let part = doc.missing_pictures.media_parts.get(&pic.media).cloned().unwrap_or_else(|| pic.media.clone());
|
||||
let item = MissingPicture { part, story: "background".into(), reason };
|
||||
if out.len() >= MAX_MISSING_PICTURES {
|
||||
truncated = true;
|
||||
} else if !out.contains(&item) {
|
||||
out.push(item);
|
||||
if !out.contains(&item) {
|
||||
if out.len() >= MAX_MISSING_PICTURES {
|
||||
truncated = true;
|
||||
if unlisted.insert(picture_hash(&item)) {
|
||||
total = total.saturating_add(1);
|
||||
}
|
||||
} else {
|
||||
total = total.saturating_add(1);
|
||||
out.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(out, truncated)
|
||||
(out, truncated, total)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::{DocLayout, MAX_MISSING_GLYPHS, MissingGlyphs};
|
||||
/// Code points in unassigned plane 4 that no face of the host covers, checked against the body
|
||||
/// font and the whole fallback chain (so the test never guesses).
|
||||
fn uncovered(n: usize) -> Vec<char> {
|
||||
crate::test_fonts();
|
||||
let db = FontDb::global();
|
||||
let primary = wordcraft_fonts::word::resolve(&Document::default().settings.minor_font, false, false).face;
|
||||
let out: Vec<char> = (0x40000u32..0x4FFF0)
|
||||
@@ -76,7 +77,7 @@ fn a_character_no_face_covers_is_counted_once_per_occurrence_in_the_body() {
|
||||
let c = uncovered(1)[0];
|
||||
let d = Document::from_text(&format!("a{c}b{c}{c}c"));
|
||||
let m = missing(&d, &lay(&d));
|
||||
assert_eq!(m, MissingGlyphs { chars: vec![(c, 3)], truncated: false });
|
||||
assert_eq!(m, MissingGlyphs::from_listed(vec![(c, 3)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -161,18 +162,6 @@ fn a_missing_glyph_is_found_in_an_equation() {
|
||||
assert_eq!(m.chars, vec![(c, 1)], "{m:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_in_many_scripts_report_nothing() {
|
||||
if std::env::var_os("CRAFT_FONTS_DIR").is_none() {
|
||||
eprintln!("CRAFT_FONTS_DIR unset: the multilingual faces aren't available, skipping");
|
||||
return;
|
||||
}
|
||||
let names = ["Łukasz Żmijewski", "Gheorghe Ștefănescu", "Krejčí", "Şahın Gül", "Llŷr Grŵn", "王小明", "김민수", "สมชาย ใจดี", "राम शर्मा"];
|
||||
let d = Document::from_text(&names.join("\n"));
|
||||
let m = missing(&d, &lay(&d));
|
||||
assert!(m.is_empty(), "{m:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn joiners_and_variation_selectors_are_never_counted() {
|
||||
let d = Document::from_text("a\u{200D}b\u{200C}c\u{FE0F}d\u{FE0E}e\u{00AD}f\u{200E}g\u{2028}h\ti");
|
||||
@@ -199,12 +188,14 @@ fn the_list_is_sorted_and_capped_at_256_distinct_code_points() {
|
||||
let m = missing(&d, &lay(&d));
|
||||
assert!(m.truncated);
|
||||
assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS);
|
||||
assert_eq!(m.total, MAX_MISSING_GLYPHS + 44, "the total counts the unlisted ones too");
|
||||
assert!(m.chars.windows(2).all(|w| w[0].0 < w[1].0), "sorted by code point");
|
||||
// Exactly the cap's worth is no truncation.
|
||||
let d = Document::from_text(&all.iter().take(MAX_MISSING_GLYPHS).collect::<String>());
|
||||
let m = missing(&d, &lay(&d));
|
||||
assert!(!m.truncated);
|
||||
assert_eq!(m.chars.len(), MAX_MISSING_GLYPHS);
|
||||
assert_eq!(m.total, MAX_MISSING_GLYPHS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -247,6 +238,7 @@ fn an_equation_run_names_each_glyphs_characters() {
|
||||
runs(&x, &mut found);
|
||||
}
|
||||
// The equation's runs (the body text "Eq: " is a run too) each carry one range per glyph.
|
||||
assert!(found.iter().any(|(t, g, _)| t.contains('\u{1D44E}') && *g == 2), "{found:?}");
|
||||
// The letters are the math italic ones where the font has them, else plain.
|
||||
assert!(found.iter().any(|(t, g, _)| (t.contains('\u{1D44E}') || t == "ab") && *g == 2), "{found:?}");
|
||||
assert!(found.iter().all(|(_, g, r)| g == r), "{found:?}");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user