diff --git a/crates/oracle/README.md b/crates/oracle/README.md index c94e08063..2f0d77aea 100644 --- a/crates/oracle/README.md +++ b/crates/oracle/README.md @@ -80,6 +80,13 @@ transparent margin is not scored as black. `cargo xtask render-oracle` drives it over the corpus. It reports; it does not gate. +A high score is a *disagreement*, not a verdict on us. The worst file of the first 250, +`bug1552113.pdf` at 84%, is Poppler painting a link's `/Border [0 0 112]` as a band over the text +the fixture exists to show; ours is the readable rendering. Look at both images before believing +the number. `--hide-annotations` separates annotation differences from page content, though the +default shows them: hiding is slightly worse overall, because it then scores Poppler for every +appearance we do draw. + ## Still to do - Acrobat itself as the reference, rather than other open renderers. diff --git a/docs/gaps.md b/docs/gaps.md index 94bd360e7..318270ee7 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -221,6 +221,18 @@ update the numbers. CI, and a first reading of a borrowed renderer is a baseline to improve on, not a bar to clear. The worst results cluster on JBIG2 `bitmap-*` files (`hayro-jbig2` carries 3 local patches) and on ZapfDingbats, which points the renderer work at something specific for the first time. +- **Read the score as a disagreement, not a verdict (2026-10-14).** The worst file of the 250, + `bug1552113.pdf` at 84%, turned out to be Poppler's defect: it paints the link's `/Border [0 0 + 112]` as a 112-unit band over the two lines of text, and the fixture is named "this text should + be visible". Ours is the readable rendering. Anything at the top of the list needs a person to + look at both images before it is treated as a bug of ours. +- **Annotations are shown, and that was measured rather than argued.** Poppler synthesises an + appearance for an annotation that has none, where we draw only what the document carries (link + borders without `/AP` are recorded as not covered under `view.render-annotation-appearances`), so + showing them scores us for a known gap. Hiding them is worse, because it then scores Poppler for + every appearance we *do* draw: over the same 250 files, shown gives mean 2.16% / p90 5.35% / 25 + files within 0.1%, hidden gives 2.18% / 5.55% / 22, with `annotation-tx3.pdf` jumping to 45.64%. + The biases nearly cancel; `--hide-annotations` tells the two apart for one file at a time. - **Evidence:** text extraction (word-F1 0.98 against `pdftotext`) and now rendering (above); qpdf checks written output on every CI run since `PDFCRAFT_ORACLES_REQUIRED`. Acrobat captures must stay local in `plan/acrobat/` (clean-room, `AGENTS.md` §1.1). diff --git a/xtask/src/gates.rs b/xtask/src/gates.rs index fdc63ba80..02c965e13 100644 --- a/xtask/src/gates.rs +++ b/xtask/src/gates.rs @@ -253,6 +253,19 @@ pub fn render_oracle(args: &[String]) -> anyhow::Result<()> { let limit: usize = flag_num(args, "--limit").unwrap_or(usize::MAX); let dpi: u32 = flag_num(args, "--dpi").unwrap_or(72); let tolerance: u8 = flag_num(args, "--tolerance").unwrap_or(8); + // Annotations are shown by default, which is not the obvious choice and was settled by + // measurement rather than argument. Poppler synthesises an appearance for an annotation that + // has none, where we draw only what the document carries (`view.render-annotation-appearances` + // records link borders without `/AP` as not covered), so showing them scores us for a known + // gap: bug1552113.pdf is 84% purely because Poppler paints a 112-unit link border over the + // text that fixture exists to show -- and our rendering, not Poppler's, is the readable one. + // + // Hiding them is worse, though, because it scores Poppler for every appearance we *do* draw. + // Over the same 250 files: shown gives mean 2.16%, p90 5.35%, 25 files within 0.1%; hidden + // gives 2.18%, 5.55% and 22, with annotation-tx3.pdf jumping to 45.64%. The two biases very + // nearly cancel, so the default stays with the fuller picture and this flag exists to tell + // the two apart when a single file's score needs explaining. + let hide_annotations = args.iter().any(|a| a == "--hide-annotations"); let dir = args.iter().position(|a| a == "--dir").and_then(|i| args.get(i + 1)).map_or_else(|| root().join("corpus/pdfjs/test/pdfs"), PathBuf::from); @@ -287,9 +300,13 @@ pub fn render_oracle(args: &[String]) -> anyhow::Result<()> { .filter(std::process::ExitStatus::success) .and_then(|_| std::fs::read(&out_pam).ok()) .and_then(|b| pdfcraft_oracle::pixels::decode_pam_gray(&b)); - let theirs = - run_capture_bytes(pdftoppm.command().args(["-singlefile", "-gray", "-f", "1", "-l", "1", "-r"]).arg(dpi.to_string()).arg(file), 30) - .and_then(|b| pdfcraft_oracle::pixels::decode_pgm(&b)); + let mut theirs_cmd = pdftoppm.command(); + theirs_cmd.args(["-singlefile", "-gray", "-f", "1", "-l", "1", "-r"]).arg(dpi.to_string()); + if hide_annotations { + theirs_cmd.arg("-hide-annotations"); + } + theirs_cmd.arg(file); + let theirs = run_capture_bytes(&mut theirs_cmd, 30).and_then(|b| pdfcraft_oracle::pixels::decode_pgm(&b)); // An oracle that cannot read a file says nothing about our fidelity, and a file we cannot // read is `xtask check`'s business, not this one's. Count each side and move on. if ours.is_none() { diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2e91afe2a..f06c09944 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -35,7 +35,7 @@ const COMMANDS: &[(&str, &str, Command)] = &[ ("text-oracle", "Compare text extraction with pdftotext over corpus/ (word F1; target median ≥ 0.97)", gates::text_oracle), ( "render-oracle", - "Compare page rendering with Poppler over corpus/ (pixel disagreement rate; --dir, --dpi, --limit, --tolerance)", + "Compare page rendering with Poppler over corpus/ (pixel disagreement rate; --dir, --dpi, --limit, --tolerance, --hide-annotations)", gates::render_oracle, ), ("screenshots", "Regenerate the README screenshots in docs/images/ and their ATTRIBUTION entries", screenshots::run),