From 907a9fa917da5174a59fae5f3c5babba051d9baa Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 24 Oct 2026 02:24:54 +0200 Subject: [PATCH] M11: monochrome downsampling: black-and-white images above 450 ppi are averaged down to 300 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PDF Optimizer and Reduce File Size downsampled colour and grey images but not black-and-white ones. Settings::mono is now MonoSettings { downsample, target_ppi, above_ppi, compression }, 300 ppi above 450 ppi and CCITT Group 4 by default (Monochrome Images ▸ Downsample to; doc_optimize mono: { downsample, ppi, above_ppi, compression }). A 1-bit image or image mask (no filter, general-purpose filters, or CCITT fax after any of them) and eight-bit grey of only black and white are averaged by area: each new pixel takes the sample covering more of it, a tie 0, so a dark line half a new pixel wide stays. They are stored as Group 4, or as 1-bit Flate when that is smaller; with Retain a resampled image keeps its kind of compression. Images with a soft mask keep their size. CCITT decoding moves from Export ▸ Images into pdfcraft-filters (decode_ccitt, CcittParams, through hayro-ccitt), and pdfcraft_cos::Document::decoded_image reads a fax image's /DecodeParms for it (undoing filters before CCITT, such as ASCII85Decode, with any /DecodeParms given by reference made direct first), so export and the optimizer share one reader, and Group 4 scans can be downsampled. A 600 ppi text page went from 79 KB to 40 KB. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BfMGHQNgdq5yf5tKZHGR3z --- Cargo.lock | 1 + Cargo.toml | 4 +- ROADMAP.md | 1 + crates/automation/src/documents.rs | 25 +- crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/automation.rs | 40 ++- crates/cos/README.md | 6 +- crates/cos/src/document.rs | 2 +- crates/cos/src/document/objects.rs | 56 +++- crates/cos/src/document/tests.rs | 77 +++++ crates/cos/src/object.rs | 2 +- crates/create/Cargo.toml | 4 +- crates/create/README.md | 13 +- crates/create/src/extract.rs | 148 +-------- crates/filters/Cargo.toml | 3 +- crates/filters/README.md | 10 +- crates/filters/src/ccitt.rs | 178 ++++++++++- crates/filters/src/lib.rs | 2 +- crates/optimize/Cargo.toml | 1 + crates/optimize/README.md | 36 ++- crates/optimize/src/images.rs | 332 ++++++++++++++------ crates/optimize/src/lib.rs | 28 +- crates/optimize/tests/optimize.rs | 158 +++++++++- crates/ui-egui/src/workflows/optimize_ui.rs | 19 +- crates/ui-egui/tests/it/create.rs | 6 +- docs/pdf-spec-parity.md | 5 +- docs/roadmap.md | 3 +- parity/acrobat-features.toml | 10 +- 28 files changed, 867 insertions(+), 305 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2427a1b0a..f9280eff06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4511,6 +4511,7 @@ dependencies = [ "pdfcraft-filters", "pdfcraft-model", "pdfcraft-render", + "proptest", "thiserror 2.0.21", "zune-jpeg", ] diff --git a/Cargo.toml b/Cargo.toml index b2380a7496..f0995622b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,8 +67,8 @@ flate2 = { version = "1.1", default-features = false, features = ["rust_backend" weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. fax = "0.2.7" -# A CCITT decoder: Export ▸ Images decodes fax images with it, and the encoder's round-trip tests -# check against it; already in the tree through hayro. +# CCITT fax decoding (pdfcraft-filters' decode_ccitt), also the independent decoder in its +# encoder's round-trip tests; already in the tree through hayro. hayro-ccitt = "0.3" # foundation diff --git a/ROADMAP.md b/ROADMAP.md index 2b2f785167..baae83997d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -205,6 +205,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-24 (M3, window move on Wayland, #3970 follow-up):** After a move or resize by PdfCraft's own title bar or edges, the button release given back to egui goes before the pointer leaving the window, so hover no longer sticks at the old place until the pointer moves. Found by CodeRabbit in review of #3970; a unit test covers it. Estimates unchanged. - **2026-10-24 (M14, #4253):** The annotation quick bar remembers Keep tool selected across restarts, including unpinning. Old and malformed settings keep safe defaults; the opt-in UI control channel reads and changes the same preference. Estimates unchanged. +- **2026-10-24 (M11, monochrome downsampling):** The PDF Optimizer and Reduce File Size average black-and-white images above 450 ppi down to 300 ppi (Monochrome Images ▸ Downsample to), Group 4 scans included, so a 600 ppi text page goes from 79 KB to 40 KB. CCITT decoding moved into `pdfcraft-filters` (`decode_ccitt`), shared by Export ▸ Images and the optimizer through `Document::decoded_image`. - **2026-10-24 (M11, black-and-white pictures):** A scan saved as a 1-bit PNG became an eight-bit grey image when made into a PDF, larger than the PNG; Create PDF now stores pictures of only black and white at one bit (Group 4 when smaller), the optimizer packs eight-bit black-and-white grey images to one bit unless the grey settings make them smaller, as they can a dithered scan, soft mask counted, and Export ▸ Images reads CCITT fax images. - **2026-10-23 (M11, monochrome compression):** The PDF Optimizer and Reduce File Size store black-and-white images (scans, image masks) losslessly as CCITT Group 4 when that is smaller, with a Monochrome Images setting in the Optimizer and `doc_optimize` `mono`. JBIG2 and monochrome downsampling are next. - **2026-10-23 (M7, #4280):** Edit text names each standard stand-in font after its face (`PCEdTimesBold`, ...) instead of one shared `PCEdHelv`. Rewriting a bold paragraph whose subset lacks an accent no longer repoints the paragraphs set earlier in a regular stand-in at Times-Bold, and the editor's frame stays on the paragraph being edited. Pages saved by older builds keep their shared name and are not repaired. Estimates unchanged. diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index 40aa4ac522..5d2c416ee5 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -280,6 +280,7 @@ impl Automation { pub(crate) fn doc_optimize(&mut self, a: &Args) -> Result { use pdfcraft_engine::optimize::{Compatibility, Compression, ImageSettings, MonoCompression, Preset}; + const MONO_KEYS: [&str; 4] = ["downsample", "ppi", "above_ppi", "compression"]; let id = self.doc(a)?.id; let before = self.doc(a)?.bytes.len(); let path = self.resolve(a.str("path")?, true)?; @@ -319,13 +320,29 @@ impl Automation { settings.gray = images("gray", settings.gray)?; if let Some(mono) = a.get("mono") { let unknown = |c: &Value| ToolError::InvalidArgs(format!("unknown mono compression {c} (ccitt_g4, retain)")); - let mono = mono.as_object().ok_or_else(|| ToolError::InvalidArgs("mono must be an object: { compression: ccitt_g4|retain }".into()))?; + let mono = mono.as_object().ok_or_else(|| { + ToolError::InvalidArgs("mono must be an object: { downsample, ppi, above_ppi, compression: ccitt_g4|retain }".into()) + })?; // A misspelled key would otherwise leave the preset's setting in place unnoticed. - if let Some(key) = mono.keys().find(|k| *k != "compression") { - return Err(ToolError::InvalidArgs(format!("unknown mono key {key:?} (compression)"))); + if let Some(key) = mono.keys().find(|k| !MONO_KEYS.contains(&k.as_str())) { + return Err(ToolError::InvalidArgs(format!("unknown mono key {key:?} ({})", MONO_KEYS.join(", ")))); } + let m = &mut settings.mono; + if let Some(d) = mono.get("downsample") { + m.downsample = d.as_bool().ok_or_else(|| ToolError::InvalidArgs(format!("mono downsample must be true or false, not {d}")))?; + } + for (key, ppi) in [("ppi", &mut m.target_ppi), ("above_ppi", &mut m.above_ppi)] { + if let Some(p) = mono.get(key) { + *ppi = p + .as_f64() + .filter(|p| p.is_finite()) + .ok_or_else(|| ToolError::InvalidArgs(format!("mono {key} must be a number, not {p}")))? + .clamp(9.0, 2400.0); + } + } + m.above_ppi = m.above_ppi.max(m.target_ppi); if let Some(c) = mono.get("compression") { - settings.mono = match c.as_str() { + m.compression = match c.as_str() { Some("ccitt_g4" | "ccitt" | "g4") => MonoCompression::CcittG4, Some("retain") => MonoCompression::Retain, _ => return Err(unknown(c)), diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 15530229fd..79555f1548 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1433,7 +1433,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { compression: ccitt_g4|retain } (default ccitt_g4: black-and-white images and image masks are stored losslessly as CCITT Group 4 when that is smaller; counted in images_recompressed). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 when that is smaller, counted in images_recompressed; retain resamples only, keeping each image's kind of compression). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index ea943eff79..e68da784d6 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -6375,6 +6375,11 @@ fn lzw_streams_are_stored_with_flate_through_tools() { /// A one-page PDF drawing a 1-bit scan of text (glyph strokes with ragged edges, as a scanner /// leaves them) of 1000 × 400 samples, stored with Flate. fn scanned_page() -> Vec { + scan_drawn(b"q 500 0 0 200 50 500 cm /Scan Do Q") +} + +/// [`scanned_page`]'s scan, drawn by `content` (which names it `/Scan`). +fn scan_drawn(content: &[u8]) -> Vec { use pdfcraft_cos::{Dict, Document, Object, SaveOptions, Stream, write_full}; let (w, h) = (1000usize, 400usize); let stride = w.div_ceil(8); @@ -6415,7 +6420,7 @@ fn scanned_page() -> Vec { x.set(b"Scan".to_vec(), Object::Ref(image)); let mut res = Dict::new(); res.set(b"XObject".to_vec(), Object::Dict(x)); - let content = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"q 500 0 0 200 50 500 cm /Scan Do Q".to_vec()))); + let content = doc.add(Object::Stream(Stream::from_raw(Dict::new(), content.to_vec()))); let mut p = Dict::new(); p.set(b"Type".to_vec(), Object::name("Page")); p.set(b"Parent".to_vec(), Object::Ref(pages)); @@ -6511,6 +6516,39 @@ fn doc_optimize_stores_black_and_white_images_as_ccitt_group_4() { } } +/// A scan drawn at 600 ppi is averaged down to 300 ppi by default. `mono: { downsample: false }` +/// keeps its size, `ppi` and `above_ppi` choose others, and values of the wrong type are refused. +#[test] +fn doc_optimize_averages_black_and_white_scans_down() { + let dir = workdir("optimize-mono-downsample"); + // 1000 × 400 samples on 120 × 48 pt: 600 ppi. + std::fs::write(dir.join("scan.pdf"), scan_drawn(b"q 120 0 0 48 50 500 cm /Scan Do Q")).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "scan.pdf" }))["doc"].as_u64().unwrap(); + let sizes = |name: &str| -> Vec<(Option, Option)> { + let written = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join(name)).unwrap())).unwrap(); + written + .object_numbers() + .into_iter() + .filter_map(|num| match &*written.get(pdfcraft_cos::ObjRef { num, generation: written.generation(num) }) { + pdfcraft_cos::Object::Stream(s) if s.dict.name(b"Subtype") == Some(b"Image") => Some((s.dict.int(b"Width"), s.dict.int(b"Height"))), + _ => None, + }) + .collect() + }; + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "small.pdf" })); + assert_eq!(r["images_resampled"].as_u64(), Some(1), "{r}"); + assert_eq!(sizes("small.pdf"), [(Some(500), Some(200))]); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "kept.pdf", "mono": { "downsample": false } })); + assert_eq!(r["images_resampled"].as_u64(), Some(0), "{r}"); + assert_eq!(sizes("kept.pdf"), [(Some(1000), Some(400))]); + ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "200.pdf", "mono": { "ppi": 200, "above_ppi": 250 } })); + assert_eq!(sizes("200.pdf"), [(Some(333), Some(133))]); + for mono in [json!({ "downsample": "yes" }), json!({ "ppi": "300" }), json!({ "above": 450 })] { + assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "mono": mono })), Err(ToolError::InvalidArgs(_))), "{mono}"); + } +} + /// A one-page PDF whose page picture is drawn as `rows` grey images one pixel row high, 600 px /// wide on a 72 pt square (600 ppi), as some PDF printers write it. fn printed_in_strips(rows: usize) -> Vec { diff --git a/crates/cos/README.md b/crates/cos/README.md index 89b6826a8a..8358a220a0 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -25,6 +25,7 @@ let real = doc.resolve(&obj); // follow one indirect refe let d = doc.dict(&obj); // Option, resolving first let data = stream.decoded()?; // filters applied on demand let data = doc.with_direct_parms(&stream).decoded()?; // /DecodeParms given by reference too +let px = doc.decoded_image(&stream)?; // an image's samples, a CCITT fax image's too doc.set(r, Object::Dict(d)); // edit: lands in the overlay let r = doc.add(obj); // append a new object @@ -88,5 +89,6 @@ recognisable in a diff and byte-stable when re-serialized. - Linearized output, Fast Web View (`core.linearization`, M11). - Structural validation against the Arlington model (`core.arlington-validation`), which will live in the reserved `arlington` crate. -- Image codecs. `pdfcraft-filters` carries the non-image filters; DCT, JPX, JBIG2 and CCITT are - still decoded through the bootstrap renderer (`core.image-filters`, M2.6). +- Image codecs. `pdfcraft-filters` carries the non-image filters and CCITT fax decoding + (`Document::decoded_image`); DCT, JPX and JBIG2 are still decoded through the bootstrap renderer + (`core.image-filters`, M2.6). diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 4c7748cc95..0d9b84b749 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex, Weak}; use crate::CosError; -use crate::object::{Dict, ObjRef, Object, Stream}; +use crate::object::{Dict, MAX_DECODED, ObjRef, Object, Stream}; use crate::parser::{EndstreamIndex, Lexer, is_whitespace, parse_indirect, parse_indirect_shared}; mod objects; diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index fb20b50cf1..bcbf8bbf41 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,7 +1,7 @@ use super::reader::find; use super::state::{LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, objstm_member, owned_bytes}; use super::xref::{Revision, XrefEntry}; -use super::{Arc, CosError, Cow, Dict, Document, Lexer, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared}; +use super::{Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared}; impl Document { pub fn bytes(&self) -> &Arc> { @@ -224,6 +224,60 @@ impl Document { self.resolve(o).as_dict().cloned() } + /// The samples of image `s`, decoded: through its filters ([`Stream::decoded`]), or, for a + /// CCITT fax image (its last filter), through the filters before it and then + /// [`pdfcraft_filters::decode_ccitt`] with its `/DecodeParms` (resolved; the dictionary of a + /// lone filter, or its entry in an array), as `CCITTFaxDecode` gives them (ISO 32000-2 §7.4.6). + /// Parameters given by reference count, for every filter ([`Document::with_direct_parms`]). + /// A fax image's `/Columns` has to be its `/Width`. Up to its `/Height` rows are decoded, + /// whatever `/Rows` says, within [`MAX_DECODED`] bytes; damaged data gives fewer, so a caller + /// checks it has the rows it needs. + pub fn decoded_image(&self, s: &Stream) -> Result, CosError> { + let s = &*self.with_direct_parms(s); + let chain = s.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?; + let Some(((pdfcraft_filters::Filter::CcittFax, _), before)) = chain.split_last() else { return s.decoded() }; + // Its place in /Filter, /Crypt included, pairs it with its /DecodeParms entry (§7.4.1). + let index = match s.dict.get(b"Filter") { + Some(Object::Array(a)) => a.len().saturating_sub(1), + _ => 0, + }; + let parms = match s.dict.get(b"DecodeParms").map(|p| self.resolve(p)).as_deref() { + Some(Object::Array(a)) => a.get(index).and_then(|p| self.dict(p)), + Some(p) if index == 0 => p.as_dict().cloned(), + _ => None, + } + .unwrap_or_default(); + let int = |key: &[u8], default: i64| parms.get(key).and_then(|v| self.resolve(v).as_int()).unwrap_or(default); + let flag = |key: &[u8], default: bool| match parms.get(key).map(|v| self.resolve(v)).as_deref() { + Some(Object::Bool(b)) => *b, + _ => default, + }; + let d = pdfcraft_filters::CcittParams::default(); + let p = pdfcraft_filters::CcittParams { + k: int(b"K", d.k), + columns: int(b"Columns", d.columns), + // As many rows as the image has: a /Rows below /Height would cut the picture short, + // and rows past /Height aren't drawn. + rows: 0, + end_of_line: flag(b"EndOfLine", d.end_of_line), + encoded_byte_align: flag(b"EncodedByteAlign", d.encoded_byte_align), + end_of_block: flag(b"EndOfBlock", d.end_of_block), + black_is_1: flag(b"BlackIs1", d.black_is_1), + }; + if s.dict.int(b"Width") != Some(p.columns) { + return Err(CosError::Filter("a fax image's /Columns aren't its /Width".into())); + } + let height = s.dict.int(b"Height").and_then(|h| u32::try_from(h).ok()).unwrap_or(0); + let stride = usize::try_from(p.columns).unwrap_or(usize::MAX).div_ceil(8).max(1); + let rows = u32::try_from(MAX_DECODED / stride).unwrap_or(u32::MAX).min(height); + // The filters before it, such as ASCII85Decode, are undone first. + let data = match before { + [] => Cow::Borrowed(&s.raw[..]), + _ => Cow::Owned(pdfcraft_filters::decode_tolerant(before, &s.raw, MAX_DECODED).map_err(|e| CosError::Filter(e.to_string()))?.0), + }; + pdfcraft_filters::decode_ccitt(&data, &p, rows).map_err(|e| CosError::Filter(e.to_string())) + } + pub fn try_get(&self, num: u32) -> Result, CosError> { match self.overlay.get(&num) { Some(Slot::Set(_, o)) => return Ok(o.clone()), diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index d8d75c4af1..49baeccec8 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -79,6 +79,83 @@ fn decode_parms_given_by_reference_are_read_once_direct() { } } +/// A CCITT fax image reads through its /DecodeParms, given directly, in an array or by +/// reference (its numbers too), with /BlackIs1, and behind other filters; one whose /Columns +/// aren't its /Width is refused, /Height rows are decoded whatever /Rows says, and other images +/// decode through their filters. +#[test] +fn fax_images_decode_with_their_parameters() { + // 10 × 3: the first five samples of each row white, the rest black. + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = pdfcraft_filters::encode_g4(&bits, 10, 3).unwrap(); + let mut doc = Document::new_empty(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(10)); + let indirect = doc.add(Object::Dict(parms.clone())); + let image = |doc: &Document, parms: Option, width: i64, height: i64| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(width)); + d.set(b"Height".to_vec(), Object::Int(height)); + d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + if let Some(p) = parms { + d.set(b"DecodeParms".to_vec(), p); + } + doc.decoded_image(&Stream::from_raw(d, coded.clone())) + }; + for p in [ + Object::Dict(parms.clone()), + Object::Array(vec![Object::Dict(parms.clone())]), + Object::Ref(indirect), + Object::Array(vec![Object::Ref(indirect)]), + ] { + assert_eq!(image(&doc, Some(p.clone()), 10, 3).unwrap(), bits, "{p:?}"); + } + let mut inverted = parms.clone(); + inverted.set(b"BlackIs1".to_vec(), Object::Bool(true)); + assert_eq!( + image(&doc, Some(Object::Dict(inverted)), 10, 3).unwrap(), + [0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000] + ); + assert_eq!(image(&doc, Some(Object::Dict(parms.clone())), 10, 2).unwrap(), bits[..4], "at most /Height rows"); + let mut by_reference = Dict::new(); + by_reference.set(b"K".to_vec(), Object::Ref(doc.add(Object::Int(-1)))); + by_reference.set(b"Columns".to_vec(), Object::Ref(doc.add(Object::Int(10)))); + assert_eq!(image(&doc, Some(Object::Dict(by_reference)), 10, 3).unwrap(), bits, "/K and /Columns by reference"); + let mut understated = parms.clone(); + understated.set(b"Rows".to_vec(), Object::Int(1)); + assert_eq!(image(&doc, Some(Object::Dict(understated)), 10, 3).unwrap(), bits, "/Height rows, whatever /Rows says"); + assert!(matches!(image(&doc, Some(Object::Dict(parms.clone())), 12, 3), Err(CosError::Filter(_))), "/Columns other than /Width"); + assert!(matches!(image(&doc, None, 10, 3), Err(CosError::Filter(_))), "the default 1728 /Columns"); + let mut plain = Dict::new(); + plain.set(b"Filter".to_vec(), Object::name("FlateDecode")); + assert_eq!(doc.decoded_image(&Stream::from_raw(plain, pdfcraft_filters::encode_flate(&bits))).unwrap(), bits); + + // Behind other filters, its parameters at its own place in the array (/Crypt counts). + let behind = |filters: &[Object], parms: &[Object], data: Vec| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(10)); + d.set(b"Height".to_vec(), Object::Int(3)); + d.set(b"Filter".to_vec(), Object::Array(filters.to_vec())); + d.set(b"DecodeParms".to_vec(), Object::Array(parms.to_vec())); + doc.decoded_image(&Stream::from_raw(d, data)) + }; + let (fax, null, own) = (Object::name("CCITTFaxDecode"), Object::Null, Object::Dict(parms.clone())); + let hex = coded.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); + let flated = pdfcraft_filters::encode_flate(&coded); + let ahx = [Object::name("AHx"), fax.clone()]; + assert_eq!(behind(&ahx, &[null.clone(), Object::Ref(indirect)], hex.clone()).unwrap(), bits, "behind ASCIIHexDecode"); + assert_eq!( + behind(&[Object::name("FlateDecode"), Object::name("CCF")], &[null.clone(), own.clone()], flated.clone()).unwrap(), + bits, + "behind Flate" + ); + assert_eq!(behind(&[Object::name("Crypt"), fax.clone()], &[null.clone(), own.clone()], coded.clone()).unwrap(), bits, "after /Crypt"); + assert!(behind(&ahx, &[own.clone(), null.clone()], hex).is_err(), "the first entry is the other filter's"); + assert!(behind(&[fax.clone(), Object::name("FlateDecode")], &[own.clone(), null.clone()], flated).is_err(), "CCITT, then Flate"); + assert!(behind(&[Object::Int(5), fax], &[null, own], coded.clone()).is_err(), "a /Filter entry that isn't a name"); +} + #[test] fn a_read_state_matches_a_clone_and_no_changed_document() { let mut doc = Document::new_empty(); diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index c9b2ff9a36..17ee7844cf 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -261,7 +261,7 @@ impl Stream { /// [`Stream::filters`], or an error when `/Filter` is present but is not a name or an /// array of names (an indirect reference, say): the data is not unfiltered, so it must not /// read as decoded. - fn readable_filters(&self) -> Result, pdfcraft_filters::FilterError> { + pub(crate) fn readable_filters(&self) -> Result, pdfcraft_filters::FilterError> { let readable = match self.dict.get(b"Filter") { None | Some(Object::Null | Object::Name(_)) => true, Some(Object::Array(a)) => a.iter().all(|f| f.as_name().is_some()), diff --git a/crates/create/Cargo.toml b/crates/create/Cargo.toml index e60e554ba8..3648c3be00 100644 --- a/crates/create/Cargo.toml +++ b/crates/create/Cargo.toml @@ -11,8 +11,6 @@ workspace = true [dependencies] pdfcraft-cos = { workspace = true } pdfcraft-filters = { workspace = true } -# Already in the graph through hayro: decodes CCITT fax images for Export ▸ Images. -hayro-ccitt.workspace = true pdfcraft-model = { workspace = true } pdfcraft-fonts = { workspace = true } # For `image::bmp_with_pixel_offset`, shared with XFA pictures. @@ -27,5 +25,7 @@ thiserror = { workspace = true } encoding_rs = "0.8" [dev-dependencies] +# An independent CCITT decoder, to check Create PDF's Group 4 images. +hayro-ccitt.workspace = true image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] } hayro-syntax = { workspace = true } diff --git a/crates/create/README.md b/crates/create/README.md index b63dc933b0..05131d9038 100644 --- a/crates/create/README.md +++ b/crates/create/README.md @@ -35,10 +35,9 @@ Soft-mask Decode intervals map samples to opacity before the mask is resized, in The existing 16-bit path still uses each sample's high byte. JPEG image Decode handling remains limited. ICC transforms and calibrated color conversion remain outside this export path. -CCITT fax images (Group 3 and 4) are decoded for export through `hayro-ccitt` with their `/DecodeParms` -(`/Columns` has to be the image's width), behind other filters such as `ASCII85Decode` too. -Byte-aligned Group 3 rows are read by their end-of-line codes, whether the fill starts each code on -a byte or ends it on one, when the data starts with one (T.4 sends one before the first row), and on -bytes otherwise, whatever `/EndOfLine` says; data that comes up short is reported. Data with codes -but none before the first row is read on bytes, and is reported or, at `/K 0`, may come out wrong. -JPEG 2000 and JBIG2 images are still reported, not exported. +CCITT fax images (Group 3 and 4) are decoded for export through `pdfcraft_cos::Document::decoded_image` with their +`/DecodeParms` (`/Columns` has to be the image's width; up to `/Height` rows, whatever `/Rows` says), behind other filters such as +`ASCII85Decode` too. Byte-aligned Group 3 rows are read by their end-of-line codes, whether the fill starts each code on a byte or +ends it on one, when the data starts with one (T.4 sends one before the first row), and on bytes otherwise, whatever `/EndOfLine` +says; data that comes up short is reported. Data with codes but none before the first row is read on bytes, and is reported or, +at `/K 0`, may come out wrong. JPEG 2000 and JBIG2 images are still reported, not exported. diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index 814b68aea2..e62eae31d2 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -4,7 +4,6 @@ //! key as alpha. Images PdfCraft can't decode yet (JPEG 2000, JBIG2, separations) are //! reported, never silently left out. -use std::borrow::Cow; use std::collections::HashSet; use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; @@ -101,149 +100,6 @@ fn filters(s: &Stream) -> Vec> { } } -/// The samples of image `s`, decoded: through its filters, or, for a CCITT fax image (its last -/// filter), through the filters before it and then a CCITT decoder, as rows of one-bit samples -/// each starting on a byte, with black as 0 unless `/BlackIs1` says 1, as `CCITTFaxDecode` gives -/// them (ISO 32000-2 §7.4.6). -fn decoded(doc: &Document, s: &Stream) -> Result, String> { - // Parameters given by reference, which the filters read only once direct. - let s = &*doc.with_direct_parms(s); - let names = filters(s); - match names.split_last() { - Some((f, outer)) if f == b"CCITTFaxDecode" || f == b"CCF" => { - // Such as ASCII85Decode, undone first (7.4.1). - let chain = s.filters(); - let data = match chain.split_last() { - Some((_, before)) if !before.is_empty() => { - Cow::Owned(pdfcraft_filters::decode_tolerant(before, &s.raw, pdfcraft_cos::MAX_DECODED).map_err(|e| e.to_string())?.0) - } - _ => Cow::Borrowed(&s.raw[..]), - }; - ccitt(doc, s, &data, decode_parms(doc, s, outer.len())) - } - _ => s.decoded().map_err(|e| e.to_string()), - } -} - -/// The `/DecodeParms` of filter `index` of a stream: the dictionary of a stream with one filter, -/// or that entry of an array (7.4.2). -fn decode_parms(doc: &Document, s: &Stream, index: usize) -> Dict { - let Some(p) = s.dict.get(b"DecodeParms").map(|p| doc.resolve(p)) else { return Dict::default() }; - let p = match &*p { - Object::Array(a) => match a.get(index) { - Some(entry) => doc.resolve(entry), - None => return Dict::default(), - }, - _ if index == 0 => p, - _ => return Dict::default(), - }; - p.as_dict().cloned().unwrap_or_default() -} - -/// [`decoded`] for a CCITT fax image: `data`, coded with parameters `p` (Table 11 defaults). -/// `/Columns` has to be the image's width; pixels past it and rows past the image's height are -/// dropped, so the output is bounded by the size the caller already checked. A stream that ends -/// early gives fewer rows, which the caller refuses. -fn ccitt(doc: &Document, s: &Stream, data: &[u8], p: Dict) -> Result, String> { - struct Rows { - out: Vec, - row: Vec, - x: usize, - columns: usize, - rows: usize, - black_is_1: bool, - } - impl Rows { - fn push(&mut self, white: bool, count: usize) { - for _ in 0..count { - if self.x < self.columns - && white != self.black_is_1 - && let Some(b) = self.row.get_mut(self.x / 8) - { - *b |= 0x80 >> (self.x % 8); - } - self.x = self.x.saturating_add(1); - } - } - } - impl hayro_ccitt::Decoder for Rows { - fn push_pixel(&mut self, white: bool) { - self.push(white, 1); - } - fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { - self.push(white, (chunks as usize).saturating_mul(8)); - } - fn next_line(&mut self) { - if self.out.len() / self.row.len().max(1) < self.rows { - self.out.extend_from_slice(&self.row); - } - self.row.fill(0); - self.x = 0; - } - } - let flag = |key: &[u8], default: bool| match p.get(key).map(|v| doc.resolve(v)).as_deref() { - Some(Object::Bool(b)) => *b, - _ => default, - }; - let int = |key: &[u8]| p.get(key).and_then(|v| doc.resolve(v).as_int()); - let k = int(b"K").unwrap_or(0); - // A row is the image's width: that was checked against the export's size limit, so a hostile - // /Columns can't size the buffers. - let columns = u32::try_from(int(b"Columns").unwrap_or(1728)).ok().filter(|c| *c > 0).ok_or("the fax image has no columns")?; - if s.dict.int(b"Width") != Some(i64::from(columns)) { - return Err("the fax image's columns aren't its width".into()); - } - // As many rows as the image has, whatever /Rows says: one below /Height would cut the picture - // short, and rows past /Height aren't drawn. - let rows = s.dict.int(b"Height").and_then(|h| u32::try_from(h).ok()).unwrap_or(0); - let stride = (columns as usize).div_ceil(8); - let decode = |aligned: bool| { - let settings = hayro_ccitt::DecodeSettings { - columns, - rows, - end_of_block: flag(b"EndOfBlock", true), - end_of_line: flag(b"EndOfLine", false), - rows_are_byte_aligned: aligned, - encoding: match k { - ..0 => hayro_ccitt::EncodingMode::Group4, - 0 => hayro_ccitt::EncodingMode::Group3_1D, - k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) }, - }, - invert_black: false, - }; - let mut out = Rows { - out: Vec::new(), - row: vec![0; stride], - x: 0, - columns: columns as usize, - rows: rows as usize, - black_is_1: flag(b"BlackIs1", false), - }; - // Rows decoded before damaged data are kept; too few rows are refused by the caller. - let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)); - out.out - }; - let aligned = flag(b"EncodedByteAlign", false); - // Group 4 reads as its parameters say, and so do rows not on bytes. - if k < 0 || !aligned { - return Ok(decode(aligned)); - } - // Group 3 rows on bytes may have end-of-line codes whatever /EndOfLine says (the filter always - // accepts them), with the fill starting each code on a byte or ending it on one (T.4's fill, as - // TIFF writes it). Read unaligned, the codes mark the rows either way; without codes, the rows - // are read on bytes. A reading that comes up short isn't tried the other way: data cut short - // would then read as other rows, and is refused instead. - Ok(decode(!starts_with_end_of_line(data))) -} - -/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill -/// included. T.4 sends one before the first row when it sends them at all, and no other code starts -/// with more than eight zeros. -fn starts_with_end_of_line(data: &[u8]) -> bool { - let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false }; - i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11 -} - /// The file for one image: its JPEG data as is, or a PNG. An image with transparency (a soft /// mask, a stencil `/Mask` or a colour-key `/Mask`) is always a PNG with that transparency as /// alpha, JPEG images included, so cut-out images keep their transparent background. @@ -291,7 +147,7 @@ fn image(doc: &Document, s: &Stream) -> Result<(&'static str, Vec), String> space = decoded; (samples, 8) } else { - let data = decoded(doc, s)?; + let data = doc.decoded_image(s).map_err(|e| e.to_string())?; let bpc = if mask { 1 } else { s.dict.int(b"BitsPerComponent").unwrap_or(8) }; let bpc = match bpc { 1 => 1, @@ -633,7 +489,7 @@ fn gray_plane(doc: &Document, m: &Stream, w: usize, h: usize) -> Option> _ => return None, }; let max = (1u32 << bpc.min(8)) - 1; - unpack(&decoded(doc, m).ok()?, mw, mh, 1, bpc)?.into_iter().map(|v| (v as u32 * 255 / max) as u8).collect() + unpack(&doc.decoded_image(m).ok()?, mw, mh, 1, bpc)?.into_iter().map(|v| (v as u32 * 255 / max) as u8).collect() } }; if let Some([lo, hi]) = decode(doc, m, 1).and_then(|pairs| pairs.first().copied()) { diff --git a/crates/filters/Cargo.toml b/crates/filters/Cargo.toml index 2ed64eb8bd..91b5d3d578 100644 --- a/crates/filters/Cargo.toml +++ b/crates/filters/Cargo.toml @@ -12,10 +12,11 @@ workspace = true [dependencies] fax.workspace = true +# CCITT fax decoding (`decode_ccitt`); already in the tree through hayro. +hayro-ccitt.workspace = true flate2.workspace = true thiserror.workspace = true [dev-dependencies] -hayro-ccitt.workspace = true proptest.workspace = true weezl.workspace = true diff --git a/crates/filters/README.md b/crates/filters/README.md index 413dff37c1..b59b170c90 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -3,7 +3,7 @@ PDF stream filters: the decoders and encoders behind `/Filter` and `/DecodeParms`. - **Layer:** L0, standalone. No dependency on any other PdfCraft crate. `cos` depends on it (architecture §3 rule 3). -- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `miniz_oxide` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder. +- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `miniz_oxide` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder; CCITT decoding (T.4 and T.6) is the `hayro-ccitt` crate's (MIT OR Apache-2.0, already in the tree through hayro), bounded here. - `#![forbid(unsafe_code)]`. Builds for `wasm32-unknown-unknown`. ## API @@ -23,6 +23,8 @@ decode_tolerant_counted(chain, data, max_output) -> Result<(Vec, bool /*part encode(filter, params, data) -> Result, FilterError> encode_flate(data) -> Vec // zlib, level 6 encode_g4(data, columns, rows) -> Result, FilterError> // 1-bit rows as CCITT Group 4; columns <= MAX_G4_COLUMNS +pub struct CcittParams { k, columns, rows, end_of_line, encoded_byte_align, end_of_block, black_is_1 } // Table 11; Default = its defaults +decode_ccitt(data, ¶ms, max_rows) -> Result, FilterError> // CCITT fax as CCITTFaxDecode gives it; columns <= MAX_CCITT_COLUMNS ``` - `chain` is applied first to last, as in a `/Filter` array. @@ -41,16 +43,16 @@ encode_g4(data, columns, rows) -> Result, FilterError> // 1-bit rows a | ASCIIHexDecode | white space ignored, odd digit padded with 0, `>` EOD (optional), other characters → Corrupt | upper case, newline every 64 digits, `>` | | ASCII85Decode | `<~` prefix, `z`, white space, `~>` (or bare `~`, or none) EOD, final partial group; `z` inside a group, a 1-char final group, overflow or bad characters → Corrupt | `z` for zero groups, newline every 75 characters, `~>` | | RunLengthDecode | 0–127 literal, 129–255 repeat, 128 EOD (optional); truncated run → Corrupt | runs of ≥2 repeat, literals up to 128, 128 | -| CCITTFaxDecode | not here (`Unsupported`; renderers decode it) | `encode_g4`: Group 4 (`/K -1`) with an end-of-block mark, 0 bits coded black so `/BlackIs1 false` decodes the same bits; rows packed MSB first, each on a new byte, padding bits and trailing bytes ignored; up to 32,767 samples a row (the encoder counts positions in 16 bits); no rows or columns, or too wide → Unsupported, too few bytes → Corrupt | +| CCITTFaxDecode | not in a chain (`Unsupported`); `decode_ccitt` decodes Group 3 and Group 4 on its own, as rows of 1-bit samples each on a new byte, black 0 unless `black_is_1`, at most `max_rows` rows (fewer with `/Rows`), the rows before any damage, up to 1,048,576 samples a row (`pdfcraft_cos::Document::decoded_image` calls it for a fax image, after the filters before it) | `encode_g4`: Group 4 (`/K -1`) with an end-of-block mark, 0 bits coded black so `/BlackIs1 false` decodes the same bits; rows packed MSB first, each on a new byte, padding bits and trailing bytes ignored; up to 32,767 samples a row (the encoder counts positions in 16 bits); no rows or columns, or too wide → Unsupported, too few bytes → Corrupt | ## Tests - Unit tests per filter with hand-computed vectors (the ISO 32000-2 LZW example, RFC 1951 stored blocks, PNG/TIFF rows). - LZW is cross-checked against `weezl` (dev-dependency) in both directions for both early-change settings. - `tests/proptests.rs`: encode→decode round trips for every encodable filter × predictor × colors × bpc × columns, two-filter chains, output-limit checks, and fuzz-style no-panic tests (random bytes, random/nonsense params, bit-flipped valid streams). -- `ccitt.rs`: Group 4 output decoded by `hayro-ccitt` (dev-dependency, an independent decoder) matches the input bits: a text-like page, padding bits, the widest row, and property tests over bitmaps of noise and of runs; any input returns rather than panics. +- `ccitt.rs`: `decode_ccitt` gives back the rows `encode_g4` was given (with `/BlackIs1`, `/Rows` and a row limit), refuses bad widths, and bounds its output for any input. Group 4 output decoded by `hayro-ccitt` (an independent decoder) matches the input bits: a text-like page, padding bits, the widest row, and property tests over bitmaps of noise and of runs; any input returns rather than panics. - `fuzz/`: a detached cargo-fuzz crate (`cargo +nightly fuzz run decode`, run from `crates/filters/`). ## Status -M1.1/M1.2 for the general-purpose filters are done. Not yet here: the image codecs (DCT via zune-jpeg, JPX via hayro-jpeg2000, JBIG2, CCITT G3/G4 decode, DCT encode), CCITT Group 3 encoding and Flate encode levels other than 6. +M1.1/M1.2 for the general-purpose filters are done. Not yet here: the image codecs (DCT via zune-jpeg, JPX via hayro-jpeg2000, JBIG2, DCT encode), CCITT Group 3 encoding and Flate encode levels other than 6. diff --git a/crates/filters/src/ccitt.rs b/crates/filters/src/ccitt.rs index 3c8018525a..731a1a6de5 100644 --- a/crates/filters/src/ccitt.rs +++ b/crates/filters/src/ccitt.rs @@ -1,5 +1,6 @@ -//! CCITT Group 4 (ITU-T T.6) encoding, for `CCITTFaxDecode` with `/K -1` (ISO 32000-2 §7.4.6). -//! The coding is the `fax` crate's; this module checks what it is given. +//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`, +//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the +//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give. use fax::encoder::Encoder; use fax::{Color, VecWriter}; @@ -49,6 +50,133 @@ pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result, Filter } } +/// The widest row [`decode_ccitt`] decodes: its row buffer stays within 128 KiB. +pub const MAX_CCITT_COLUMNS: u32 = 1 << 20; + +/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the +/// table's defaults. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CcittParams { + /// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed. + pub k: i64, + pub columns: i64, + /// `/Rows`: 0 when not given. + pub rows: i64, + pub end_of_line: bool, + pub encoded_byte_align: bool, + pub end_of_block: bool, + pub black_is_1: bool, +} + +impl Default for CcittParams { + fn default() -> Self { + CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false } + } +} + +/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most +/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`). +/// +/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by +/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a +/// caller that needs a number of rows checks it got them. +/// +/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes, +/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4 +/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever +/// `end_of_line` says. Group 4 is read as its parameters say. +/// +/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`FilterError::Unsupported`]). +pub fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result, FilterError> { + let columns = u32::try_from(p.columns) + .ok() + .filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c)) + .ok_or_else(|| FilterError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?; + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + if rows == 0 { + return Ok(Vec::new()); + } + let settings = hayro_ccitt::DecodeSettings { + columns, + rows, + end_of_block: p.end_of_block, + end_of_line: p.end_of_line, + // Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter + // always accepts them), with the fill starting each code on a byte or ending it on one (T.4's + // fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without + // codes, the rows are read on bytes. A reading that comes up short isn't tried the other + // way: data cut short would then read as other rows. + rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)), + encoding: match p.k { + ..0 => hayro_ccitt::EncodingMode::Group4, + 0 => hayro_ccitt::EncodingMode::Group3_1D, + k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) }, + }, + invert_black: false, + }; + // u32 to usize: wasm32 and up have at least 32-bit usize. + let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1); + // The rows decoded before damaged data are kept. + let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)); + Ok(out.out) +} + +/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill +/// included. T.4 sends one before the first row when it sends them at all, and no other code starts +/// with more than eight zeros. +fn starts_with_end_of_line(data: &[u8]) -> bool { + let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false }; + i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11 +} + +/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at +/// most `rows` rows. Pixels past a row's end and rows past the last are dropped. +struct Rows { + out: Vec, + row: Vec, + x: usize, + columns: usize, + rows: usize, + done: usize, + black_is_1: bool, +} + +impl Rows { + fn new(columns: usize, rows: usize, black_is_1: bool) -> Self { + Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 } + } + + /// `count` pixels, white or black. + fn push(&mut self, white: bool, count: usize) { + let end = self.x.saturating_add(count); + if white != self.black_is_1 { + for x in self.x..end.min(self.columns) { + if let Some(b) = self.row.get_mut(x / 8) { + *b |= 0x80 >> (x % 8); + } + } + } + self.x = end; + } +} + +impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.push(white, 1); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.push(white, (chunks as usize).saturating_mul(8)); + } + fn next_line(&mut self) { + if self.done < self.rows { + self.out.extend_from_slice(&self.row); + self.done += 1; + } + self.row.fill(0); + self.x = 0; + } +} + #[cfg(test)] mod tests { use super::*; @@ -159,8 +287,40 @@ mod tests { data } + fn g4(columns: u32) -> CcittParams { + CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() } + } + + #[test] + fn decoding_gives_the_rows_back() { + // 3 columns, padded with anything: the decoded padding is 0. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010]; + let coded = encode_g4(&data, 3, 3).unwrap(); + assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]); + // /BlackIs1 gives black as 1. + let p = CcittParams { black_is_1: true, ..g4(3) }; + assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]); + // At most /Rows rows, and at most max_rows. + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2); + assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1); + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows"); + assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]); + // Damaged data gives the rows before the damage; none here. + assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3); + } + + #[test] + fn what_it_cannot_decode_is_an_error() { + for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] { + let p = CcittParams { columns, ..CcittParams::default() }; + assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(FilterError::Unsupported(_))), "{columns}"); + } + assert_eq!(CcittParams::default().columns, 1728, "Table 11"); + } + proptest::proptest! { - /// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder. + /// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder + /// and through `decode_ccitt`. #[test] fn bitmaps_round_trip( columns in 1u32..400, @@ -169,14 +329,20 @@ mod tests { noise in proptest::bool::ANY, ) { let data = bitmap(columns, rows, &runs, noise); - let g4 = encode_g4(&data, columns, rows).unwrap(); - proptest::prop_assert_eq!(decode(&g4, columns, rows), unpack(&data, columns, rows)); + let coded = encode_g4(&data, columns, rows).unwrap(); + proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows)); + let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap(); + proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows)); } /// Whatever it is given, it returns. #[test] - fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..40_000, rows in 0u32..80) { + fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..40_000, rows in 0u32..80, k in -2i64..3) { let _ = encode_g4(&data, columns, rows); + let p = CcittParams { k, columns: i64::from(columns), ..CcittParams::default() }; + if let Ok(out) = decode_ccitt(&data, &p, rows) { + proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize); + } } } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index b9461435eb..63daefc4ea 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -24,7 +24,7 @@ mod runlength; use std::borrow::Cow; -pub use ccitt::{MAX_G4_COLUMNS, encode_g4}; +pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt, encode_g4}; /// A PDF stream filter, identified by its `/Filter` name. #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/crates/optimize/Cargo.toml b/crates/optimize/Cargo.toml index 456ab4c991..d1a759d468 100644 --- a/crates/optimize/Cargo.toml +++ b/crates/optimize/Cargo.toml @@ -24,3 +24,4 @@ zune-jpeg = "0.5" [dev-dependencies] pdfcraft-render = { workspace = true } +proptest.workspace = true diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 1f184f0ffb..ba71eb38f5 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -34,18 +34,30 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going alone, since readers differ on which wins. Samples are read with `/DecodeParms` given by reference too, the numbers inside included (`Document::with_direct_parms`), here and below: without them a predictor's tag bytes would shift every row of an image written back. -- **Monochrome images** (`Settings::mono`, CCITT Group 4 by default): a 1-bit image with one - colour component (grey, indexed, a separation) or an image mask, stored with no filter or with - general-purpose ones, is stored again as CCITT Group 4 (`pdfcraft_filters::encode_g4`) when - that is smaller as stored. An eight-bit grey image of only 0 and 255 is packed to one bit first - (not an indexed image, whose samples are palette indices, nor one with a colour-key `/Mask`), - unless the grey settings make it smaller still, its soft mask counted (resampled with it there, - kept as it is here), as they can a dithered scan by downsampling it to JPEG. It is lossless: the samples decode to the same bits, so the colour - space, decode array and masks stay. Group 4 wins on scans, whose ragged edges defeat Flate; - on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Not yet: - JBIG2, monochrome downsampling, inline images, soft masks and `/Mask` images, and rows wider - than 32,767 samples. Redaction doesn't decode Group 4 yet: a mark over part of an image stored - this way removes the whole image (nothing leaks). +- **Monochrome images** (`Settings::mono`, `MonoSettings::REDUCE` by default: to 300 ppi above + 450 ppi, CCITT Group 4). Black-and-white images are a 1-bit image with one colour component + (grey, indexed, a separation) or an image mask, stored with no filter, with general-purpose ones + or with CCITT fax after any of those (decoded by `Document::decoded_image`), and an eight-bit grey image of + only 0 and 255, packed to one bit (not an indexed image, whose samples are palette indices, nor + one with a colour-key `/Mask`) unless the grey settings make it smaller still, its soft mask counted + (resampled with it there, kept as it is here), as they can a dithered scan by downsampling it to + JPEG. + - **Downsampling:** one whose effective resolution is above `above_ppi` is averaged down to + `target_ppi`: each new pixel takes the sample covering more of its area, a 0 (black in grey) + on a tie, so a dark line half a new pixel wide stays. Not one with a soft mask, whose own size + would have to follow. On a 600 ppi text page, Group 4 at 300 ppi is half the size. + - **Compression:** CCITT Group 4 (`pdfcraft_filters::encode_g4`), or 1-bit Flate when that is + smaller, which only a resampled image or one packed from eight bits may be: a 1-bit image that + is only stored again becomes Group 4 or stays. A CCITT image is only decoded to be resampled, + unless another filter such as `ASCII85Decode` wraps it, which storing it again drops. With Retain, a resampled image keeps its kind of + compression (CCITT, or Flate) and nothing else changes; eight-bit grey is then left to the grey + settings. Every change is kept only when it is smaller as stored. + - Apart from the downsampling it is lossless: the samples keep their meaning, so the colour + space, decode array and `/Mask` stay. Group 4 wins on scans, whose ragged edges defeat Flate; + on clean computer-drawn 1-bit art Flate can be smaller. Not yet: JBIG2, Group 3 to Group 4 + without resampling, inline images, and Group 4 for rows wider than 32,767 samples. Redaction + doesn't decode Group 4 yet: a mark over part of an image stored this way removes the whole + image (nothing leaks). - **Discard objects:** thumbnails, alternate images, document tags, print settings. - **Clean up:** Flate for streams with no filter; for streams with an outer ASCII85 or ASCIIHex layer, that layer is removed when the result is smaller (the inner filters keep their encoded diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 8550affb5c..f6fd5e5770 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::{Matrix, Pieces}; use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; -use crate::{Compression, ImageSettings, MonoCompression, OptimizeError, Report, Settings, Stage}; +use crate::{Compression, ImageSettings, MonoCompression, MonoSettings, OptimizeError, Report, Settings, Stage}; /// Images with more pixels than this are left alone (memory). const MAX_PIXELS: u64 = 80_000_000; @@ -562,102 +562,172 @@ fn one_component(doc: &Document, cs: &Object) -> bool { } } -/// A black-and-white image (one bit per pixel, or an image mask) stored again as CCITT Group 4, -/// if `compression` asks for that and the result is smaller as stored. Only images whose samples -/// PdfCraft decodes (no filter, or general-purpose filters: image codecs don't decode here) -/// qualify. The samples decode to the same bits, so the colour space, decode array and masks stay -/// as they are. -fn mono(doc: &Document, s: &Stream, compression: MonoCompression) -> Option { - if compression != MonoCompression::CcittG4 { - return None; - } +/// The samples of a black-and-white image, as rows of 1-bit samples each starting on a new byte, +/// with its width and height, and whether they were packed from eight bits. Two kinds count: +/// - one bit per pixel in one colour component, or an image mask, stored with no filter, with +/// general-purpose ones, or with CCITT fax last ([`Document::decoded_image`]); +/// - with `eight_bit`, eight-bit grey whose every sample is 0 or 255. A decode array maps 0 and 255 +/// at eight bits as it maps 0 and 1 at one bit, so it stays. Left out: indexed images (their +/// samples are palette indices) and a colour-key `/Mask` (its ranges are in eight-bit values). +fn one_bit(doc: &Document, s: &Stream, eight_bit: bool) -> Option<(Vec, u32, u32, bool)> { let d = &s.dict; - let bits = d.int(b"BitsPerComponent"); - let qualifies = if matches!(d.get(b"ImageMask"), Some(Object::Bool(true))) { - bits.is_none_or(|b| b == 1) - } else { - bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs)) - }; - if !qualifies { - return None; - } - let (w, h) = (u32::try_from(d.int(b"Width")?).ok()?, u32::try_from(d.int(b"Height")?).ok()?); - if w == 0 || h == 0 || w > pdfcraft_filters::MAX_G4_COLUMNS || u64::from(w) * u64::from(h) > MAX_PIXELS { - return None; - } - // Through its parameters given by reference too, as `pixels` reads, and no further than its - // rows can need. - let bytes = usize::try_from(w).ok()?.div_ceil(8).checked_mul(usize::try_from(h).ok()?)?; - let g4 = pdfcraft_filters::encode_g4(&doc.with_direct_parms(s).decoded_within(allowance(bytes, h)?).ok()?, w, h).ok()?; - let mut new = d.clone(); - for k in [&b"Filter"[..], b"DecodeParms", b"Length", b"DL"] { - new.remove(k); - } - let mut parms = Dict::new(); - parms.set(b"K".to_vec(), Object::Int(-1)); - parms.set(b"Columns".to_vec(), Object::Int(i64::from(w))); - parms.set(b"Rows".to_vec(), Object::Int(i64::from(h))); - new.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); - new.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - let new = Stream::from_raw(new, g4); - // Smaller as stored, counting the larger dictionary: a tiny image can lose by it. - (crate::stored_len(&new)? < crate::stored_len(s)?).then_some(new) -} - -/// An eight-bit grey image whose every sample is 0 or 255, stored at one bit per pixel and then as -/// `compression` asks (see [`mono`]), if that is smaller as stored. Lossless: a decode array maps -/// 0 and 255 at eight bits as it maps 0 and 1 at one bit, so it stays. Left alone: indexed images -/// (their samples are palette indices), a colour-key `/Mask` (its ranges are in eight-bit values), -/// and everything when `compression` is Retain: such an image is then a grey image like any other, -/// which the grey settings handle as they did before. -fn bilevel_grey(doc: &Document, s: &Stream, compression: MonoCompression) -> Option { - if compression == MonoCompression::Retain { - return None; - } - let d = &s.dict; - let indexed = |cs: &Object| match &*doc.resolve(cs) { - Object::Array(a) => matches!(a.first().and_then(Object::as_name), Some(b"Indexed" | b"I")), - _ => false, - }; - let colour_key = d.get(b"Mask").is_some_and(|m| doc.resolve(m).as_array().is_some()); - if d.int(b"BitsPerComponent") != Some(8) - || matches!(d.get(b"ImageMask"), Some(Object::Bool(true))) - || colour_key - || !d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs) && !indexed(cs)) - { - return None; - } // Through u32 first, as everywhere here, so the product can't overflow. let (w, h) = (u32::try_from(d.int(b"Width")?).ok()?, u32::try_from(d.int(b"Height")?).ok()?); if w == 0 || h == 0 || u64::from(w) * u64::from(h) > MAX_PIXELS { return None; } - let (w, h) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); + let (wu, hu) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); + let stride = wu.div_ceil(8); + let bits = d.int(b"BitsPerComponent"); + let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); + if (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) { + // A fax image through its decoder; any other no further than its rows can need. + let len = stride.checked_mul(hu)?; + let mut data = match s.filters().last() { + Some((pdfcraft_filters::Filter::CcittFax, _)) => doc.decoded_image(s), + _ => doc.with_direct_parms(s).decoded_within(allowance(len, h)?), + } + .ok()?; + if data.len() < len { + return None; + } + data.truncate(len); + return Some((data, w, h, false)); + } + let indexed = |cs: &Object| match &*doc.resolve(cs) { + Object::Array(a) => matches!(a.first().and_then(Object::as_name), Some(b"Indexed" | b"I")), + _ => false, + }; + let colour_key = d.get(b"Mask").is_some_and(|m| doc.resolve(m).as_array().is_some()); + if !eight_bit || bits != Some(8) || mask || colour_key || !d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs) && !indexed(cs)) { + return None; + } // Through its parameters given by reference too: without a PNG predictor's tag bytes taken // off, rows of 0 and 255 would still pass below, each shifted by a byte. - let samples = doc.with_direct_parms(s).decoded_within(allowance(w.checked_mul(h)?, u32::try_from(h).ok()?)?).ok()?; - let samples = samples.get(..w.checked_mul(h)?)?; - let stride = w.div_ceil(8); - let mut packed = vec![0u8; stride.checked_mul(h)?]; + let samples = doc.with_direct_parms(s).decoded_within(allowance(wu.checked_mul(hu)?, h)?).ok()?; + let samples = samples.get(..wu.checked_mul(hu)?)?; + let mut packed = vec![0u8; stride.checked_mul(hu)?]; for (i, v) in samples.iter().enumerate() { match v { 0 => {} 255 => { - if let Some(b) = packed.get_mut((i / w) * stride + (i % w) / 8) { - *b |= 0x80 >> (i % w % 8); + if let Some(b) = packed.get_mut((i / wu) * stride + (i % wu) / 8) { + *b |= 0x80 >> (i % wu % 8); } } _ => return None, } } - let mut one_bit = d.clone(); - for k in [&b"Filter"[..], b"DecodeParms", b"Length", b"DL"] { - one_bit.remove(k); + Some((packed, w, h, true)) +} + +/// `w` × `h` 1-bit rows (each starting on a new byte) averaged down to `nw` × `nh`, no larger: +/// each new pixel takes the sample that covers more of its area, and a 0 when the two tie. A 0 is +/// black in grey and the painted part of an image mask, so a dark line half a new pixel wide stays. +/// `None` when the rows are short or the new size isn't smaller in both directions or empty. +pub(crate) fn downsample_bits(data: &[u8], w: u32, h: u32, nw: u32, nh: u32) -> Option> { + if nw == 0 || nh == 0 || nw > w || nh > h { + return None; } - one_bit.set(b"BitsPerComponent".to_vec(), Object::Int(1)); - let one_bit = Stream::flate(one_bit, &packed); - let best = mono(doc, &one_bit, compression).unwrap_or(one_bit); - (crate::stored_len(&best)? < crate::stored_len(s)?).then_some(best) + let (stride, nstride) = (usize::try_from(w).ok()?.div_ceil(8), usize::try_from(nw).ok()?.div_ceil(8)); + let data = data.get(..stride.checked_mul(usize::try_from(h).ok()?)?)?; + let mut out = vec![0u8; nstride.checked_mul(usize::try_from(nh).ok()?)?]; + // In units of 1/nw of a source column, source column x spans [x·nw, (x+1)·nw) and new column i + // spans [i·w, (i+1)·w); rows likewise with h and nh. A new pixel's area is then w·h. + let (w, h, nw, nh) = (u64::from(w), u64::from(h), u64::from(nw), u64::from(nh)); + let overlap = |a0: u64, a1: u64, b0: u64, b1: u64| a1.min(b1).saturating_sub(a0.max(b0)); + // Per source column, the area its 1 samples cover in the new row. + let mut ones = vec![0u64; usize::try_from(w).ok()?]; + for j in 0..nh { + ones.fill(0); + let (y0, y1) = (j * h, (j + 1) * h); + for y in y0 / nh..y1.div_ceil(nh).min(h) { + let weight = overlap(y0, y1, y * nh, (y + 1) * nh); + let row = data.get(usize::try_from(y).ok()? * stride..)?; + for (x, sum) in ones.iter_mut().enumerate() { + if row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0) { + *sum += weight; + } + } + } + let out_row = out.get_mut(usize::try_from(j).ok()? * nstride..)?; + for i in 0..nw { + let (x0, x1) = (i * w, (i + 1) * w); + let covered: u64 = (x0 / nw..x1.div_ceil(nw).min(w)) + .map(|x| overlap(x0, x1, x * nw, (x + 1) * nw) * usize::try_from(x).ok().and_then(|x| ones.get(x)).copied().unwrap_or(0)) + .sum(); + // More than half of w·h, which can't overflow (w·h < 2^64 for u32 sizes). + if covered > w * h / 2 { + let i = usize::try_from(i).ok()?; + *out_row.get_mut(i / 8)? |= 0x80 >> (i % 8); + } + } + } + Some(out) +} + +/// A black-and-white image (see [`one_bit`]) resampled and stored again per `settings`, if that is +/// smaller as stored; `ppi` is its effective resolution. Also says whether it was resampled. +/// +/// - **Resampled** ([`downsample_bits`]) when its resolution is above `above_ppi`, to `target_ppi`, +/// unless it has a soft mask, whose own size would then have to follow. +/// - **Stored** as CCITT Group 4 with `CcittG4`, or as 1-bit Flate when that is smaller, which only +/// a resampled image or one packed from eight bits may be: a 1-bit image that is only stored again +/// becomes Group 4 or stays. With `Retain`, a resampled image keeps its kind of compression +/// (CCITT, or Flate), and nothing else changes: eight-bit grey stays a grey image like any +/// other, which the grey settings handle. +/// +/// The samples keep their meaning, so the colour space, decode array and `/Mask` stay as they are. +fn mono(doc: &Document, s: &Stream, ppi: f64, settings: &MonoSettings) -> Option<(Stream, bool)> { + let d = &s.dict; + let downsample = settings.downsample && ppi.is_finite() && ppi > settings.above_ppi && settings.target_ppi > 0.0 && !d.contains(b"SMask"); + let retain = settings.compression == MonoCompression::Retain; + let chain = s.filters(); + // Its kind of compression, which Retain keeps, behind any other filters. + let ccitt = matches!(chain.last(), Some((pdfcraft_filters::Filter::CcittFax, _))); + // A fax image is only worth decoding to resample it; behind ASCII85Decode, say, storing it + // again as Group 4 also drops that filter. + let fax_alone = ccitt && chain.len() == 1; + if (retain || fax_alone) && !downsample { + return None; + } + let (data, w, h, eight_bit) = one_bit(doc, s, !retain)?; + let k = settings.target_ppi / ppi; + // At most `w` × `h`, as k < 1: no overflow. + let (nw, nh) = + if downsample && k < 1.0 { (((f64::from(w) * k).round() as u32).max(1), ((f64::from(h) * k).round() as u32).max(1)) } else { (w, h) }; + let resampled = (nw, nh) != (w, h); + if (retain || fax_alone) && !resampled { + return None; + } + let data = if resampled { downsample_bits(&data, w, h, nw, nh)? } else { data }; + let mut dict = d.clone(); + for key in [&b"Filter"[..], b"DecodeParms", b"Length", b"DL"] { + dict.remove(key); + } + dict.set(b"Width".to_vec(), Object::Int(i64::from(nw))); + dict.set(b"Height".to_vec(), Object::Int(i64::from(nh))); + if eight_bit { + dict.set(b"BitsPerComponent".to_vec(), Object::Int(1)); + } + let g4 = if settings.compression == MonoCompression::CcittG4 || (retain && ccitt) { + pdfcraft_filters::encode_g4(&data, nw, nh).ok().map(|coded| { + let mut dict = dict.clone(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(nw))); + parms.set(b"Rows".to_vec(), Object::Int(i64::from(nh))); + dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + Stream::from_raw(dict, coded) + }) + } else { + None + }; + let flate = ((resampled || eight_bit) && !(retain && ccitt)).then(|| Stream::flate(dict, &data)); + // Smaller as stored, counting the dictionary: a tiny image can lose by Group 4's parameters. + let best = [g4, flate].into_iter().flatten().filter_map(|c| Some((crate::stored_len(&c)?, c))).min_by_key(|(len, _)| *len)?; + (best.0 < crate::stored_len(s)?).then_some((best.1, resampled)) } pub(crate) fn run( @@ -686,19 +756,28 @@ pub(crate) fn run( let obj = doc.get(r); let Object::Stream(s) = &*obj else { continue }; report.images += 1; - if let Some(new) = mono(doc, s, settings.mono) { + // Packed from more bits a sample, it is lossless at one bit, but the grey settings can + // make it smaller still (below). + let packed_from_more = s.dict.int(b"BitsPerComponent").is_some_and(|b| b > 1); + let (packed, mono) = match mono(doc, s, ppi, &settings.mono) { + Some(m) if packed_from_more => (Some(m), None), + m => (None, m), + }; + if let Some((new, resampled)) = mono { report.image_bytes_before += s.raw.len(); report.image_bytes_after += new.raw.len(); - report.images_recompressed += 1; + if resampled { + report.images_resampled += 1; + } else { + report.images_recompressed += 1; + } doc.set(r, Object::Stream(new)); continue; } - let packed = bilevel_grey(doc, s, settings.mono); let n = s.dict.get(b"ColorSpace").and_then(|cs| components(doc, cs)); let processed = n.and_then(|n| process(doc, s, ppi, if n == 1 { &settings.gray } else { &settings.color }, n)); - // An image packed from eight bits is lossless at one bit, but the grey settings can make it - // smaller still (a dithered scan, downsampled and JPEG-compressed): the smaller wins, - // counting a soft mask resampled with it. + // A dithered scan, say, downsampled and JPEG-compressed: the smaller wins, counting a soft + // mask resampled with it. let smask_len = |smask: Option<&(ObjRef, Stream)>| match smask { Some((_, m)) => Some(m.raw.len()), None => s.dict.reference(b"SMask").map(|m| match &*doc.get(m) { @@ -707,13 +786,17 @@ pub(crate) fn run( }), }; let total = |new: &Stream, smask: Option<&(ObjRef, Stream)>| Some(crate::stored_len(new)?.saturating_add(smask_len(smask).unwrap_or(0))); - let packed_wins = packed.as_ref().is_some_and(|p| { + let packed_wins = packed.as_ref().is_some_and(|(p, _)| { processed.as_ref().is_none_or(|(new, smask)| total(p, None).zip(total(new, smask.as_ref())).is_some_and(|(a, b)| a <= b)) }); - if let Some(new) = packed.filter(|_| packed_wins) { + if let Some((new, resampled)) = packed.filter(|_| packed_wins) { report.image_bytes_before += s.raw.len(); report.image_bytes_after += new.raw.len(); - report.images_recompressed += 1; + if resampled { + report.images_resampled += 1; + } else { + report.images_recompressed += 1; + } doc.set(r, Object::Stream(new)); continue; } @@ -788,3 +871,76 @@ mod tests { assert!(data.is_none_or(|d| d.len() <= MAX_PAGE_DECODED + 300)); } } + +#[cfg(test)] +mod downsample_tests { + use super::downsample_bits; + + /// [`downsample_bits`] the slow way: each new pixel's area against every source pixel's. + fn naive(data: &[u8], w: u32, h: u32, nw: u32, nh: u32) -> Vec { + let (stride, nstride) = ((w as usize).div_ceil(8), (nw as usize).div_ceil(8)); + let (w, h, nw, nh) = (u64::from(w), u64::from(h), u64::from(nw), u64::from(nh)); + let mut out = vec![0u8; nstride * nh as usize]; + for j in 0..nh { + for i in 0..nw { + let mut ones = 0; + for y in 0..h { + for x in 0..w { + let across = ((i + 1) * w).min((x + 1) * nw).saturating_sub((i * w).max(x * nw)); + let down = ((j + 1) * h).min((y + 1) * nh).saturating_sub((j * h).max(y * nh)); + if data[y as usize * stride + x as usize / 8] & (0x80 >> (x % 8)) != 0 { + ones += across * down; + } + } + } + if 2 * ones > w * h { + out[j as usize * nstride + i as usize / 8] |= 0x80 >> (i % 8); + } + } + } + out + } + + #[test] + fn each_new_pixel_takes_the_sample_covering_more_of_it() { + // 4 × 2 to 2 × 1: three 1s of four on the left, none on the right. + assert_eq!(downsample_bits(&[0b1100_0000, 0b1000_0000], 4, 2, 2, 1), Some(vec![0b1000_0000])); + // A tie gives 0: black in grey, so a black line half a new pixel wide stays (the second + // of four white columns is black); a white one that thin on black doesn't. + assert_eq!(downsample_bits(&[0b1011_0000, 0b1011_0000], 4, 2, 2, 1), Some(vec![0b0100_0000])); + assert_eq!(downsample_bits(&[0b0100_0000, 0b0100_0000], 4, 2, 2, 1), Some(vec![0])); + // The same size gives the same bits; padding bits are 0. + assert_eq!(downsample_bits(&[0b1010_1111], 3, 1, 3, 1), Some(vec![0b1010_0000])); + // 3 to 2 across: the middle column counts half to each new one. + assert_eq!(downsample_bits(&[0b1100_0000], 3, 1, 2, 1), Some(vec![0b1000_0000])); + } + + #[test] + fn what_it_cannot_resample_is_none() { + assert_eq!(downsample_bits(&[0; 2], 4, 2, 5, 1), None, "wider"); + assert_eq!(downsample_bits(&[0; 2], 4, 2, 2, 3), None, "taller"); + assert_eq!(downsample_bits(&[0; 2], 4, 2, 0, 1), None); + assert_eq!(downsample_bits(&[0; 1], 4, 2, 2, 1), None, "short rows"); + assert_eq!(downsample_bits(&[], u32::MAX, u32::MAX, 1, 1), None); + } + + proptest::proptest! { + /// Any bitmap, to any size no larger, gives what the slow way gives; all 1s stay all 1s. + #[test] + fn it_matches_the_slow_way( + w in 1u32..40, + h in 1u32..16, + nw_part in 0.0f64..1.0, + nh_part in 0.0f64..1.0, + seed in proptest::collection::vec(0u8..=255, 1..64), + ) { + let (nw, nh) = (1 + (f64::from(w - 1) * nw_part) as u32, 1 + (f64::from(h - 1) * nh_part) as u32); + let len = (w as usize).div_ceil(8) * h as usize; + let data: Vec = seed.iter().cycle().take(len).copied().collect(); + proptest::prop_assert_eq!(downsample_bits(&data, w, h, nw, nh), Some(naive(&data, w, h, nw, nh))); + let white = downsample_bits(&vec![0xFF; len], w, h, nw, nh).unwrap(); + proptest::prop_assert_eq!(white, naive(&vec![0xFF; len], w, h, nw, nh)); + proptest::prop_assert!(naive(&vec![0xFF; len], w, h, nw, nh).iter().any(|b| *b != 0)); + } + } +} diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index f4f7baf988..e911055941 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -10,8 +10,9 @@ //! A new image replaces the old one only if it is smaller. Images PdfCraft can't decode //! faithfully (CMYK and other colour spaces, masks, decode arrays other than the default, //! JPEG 2000, JBIG2, CCITT, other than 8 bits, JPEGs whose Adobe marker and `/ColorTransform` -//! disagree) are left alone. Black-and-white (1-bit) images are stored again as CCITT Group 4, -//! losslessly, when that is smaller. +//! disagree) are left alone. Black-and-white images (1-bit, CCITT fax among them, and eight-bit +//! grey of only black and white) above a threshold are averaged down to a target resolution, and +//! are stored as CCITT Group 4 when that is smaller. //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted @@ -128,6 +129,23 @@ pub enum MonoCompression { Retain, } +/// How black-and-white images (1-bit, CCITT fax among them, and eight-bit grey of only black +/// and white), and image masks, are resampled and stored. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MonoSettings { + /// Average images whose effective resolution is above `above_ppi` down to `target_ppi`. + pub downsample: bool, + pub target_ppi: f64, + pub above_ppi: f64, + pub compression: MonoCompression, +} + +impl MonoSettings { + /// Reduce File Size and the presets: to 300 ppi above 450 ppi, CCITT Group 4. Text in a scan + /// stays sharp at 300 ppi. + pub const REDUCE: MonoSettings = MonoSettings { downsample: true, target_ppi: 300.0, above_ppi: 450.0, compression: MonoCompression::CcittG4 }; +} + /// Acrobat's JPEG quality steps. pub const QUALITIES: [(&str, u8); 5] = [("Minimum", 25), ("Low", 45), ("Medium", 60), ("High", 80), ("Maximum", 95)]; @@ -157,8 +175,8 @@ pub struct Settings { pub merge_fragments: bool, pub color: ImageSettings, pub gray: ImageSettings, - /// Black-and-white (1-bit) images, and image masks. - pub mono: MonoCompression, + /// Black-and-white images, and image masks. + pub mono: MonoSettings, pub discard_thumbnails: bool, pub discard_alternate_images: bool, pub discard_tags: bool, @@ -193,7 +211,7 @@ impl Default for Settings { merge_fragments: true, color: ImageSettings::REDUCE, gray: ImageSettings::REDUCE, - mono: MonoCompression::CcittG4, + mono: MonoSettings::REDUCE, discard_thumbnails: true, discard_alternate_images: true, discard_tags: false, diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 9e82858776..e904088d24 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -5,7 +5,8 @@ use std::sync::Arc; use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; use pdfcraft_filters::{Filter, Params}; use pdfcraft_optimize::{ - Compatibility, Compression, ImageSettings, MonoCompression, OptimizeError, Settings, SpaceCategory, audit_space, effective_resolutions, optimize, + Compatibility, Compression, ImageSettings, MonoCompression, MonoSettings, OptimizeError, Settings, SpaceCategory, audit_space, + effective_resolutions, optimize, }; fn image(doc: &mut Document, w: u32, h: u32, n: usize, jpeg: bool, smask: Option) -> ObjRef { @@ -394,7 +395,7 @@ fn black_and_white_images_become_ccitt_group_4() { let untouched = [noise, rgb, wide, tiny, named].map(|r| raw(&doc, r)); let mut kept = doc.clone(); - let r = optimize(&mut kept, &Settings { mono: MonoCompression::Retain, ..Settings::default() }).unwrap(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..Settings::default() }).unwrap(); assert_eq!(r.images_recompressed, 0, "Retain keeps every image as it is"); assert!([gray, mask, indexed, predicted].iter().all(|i| raw(&kept, *i) == raw(&doc, *i))); @@ -456,7 +457,16 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let past_limit = mono_image(&mut doc, 10_000, 8_001, &bilevel(10_000, 8_001, false), None, &grey); let all = [icc_1, icc_3, spot, two_spots, mask_1, mask_8, at_limit, past_limit]; let names: Vec = (0..all.len()).map(|i| format!("Im{i}")).collect(); - let content: String = (0..all.len()).map(|i| format!("q 200 0 0 80 {} {} cm /Im{i} Do Q ", 6 + 200 * (i % 3), 690 - 100 * (i / 3))).collect(); + // 200 × 80 pt is 432 ppi, 1800 × 1440 pt 400 ppi: below where monochrome downsampling starts. + let content: String = (0..all.len()) + .map(|i| { + if i < 6 { + format!("q 200 0 0 80 {} {} cm /Im{i} Do Q ", 6 + 200 * (i % 3), 690 - 100 * (i / 3)) + } else { + format!("q 1800 0 0 1440 0 0 cm /Im{i} Do Q ") + } + }) + .collect(); page(&mut doc, &names.iter().map(String::as_str).zip(all).collect::>(), &content); let converted = [icc_1, spot, mask_1, at_limit]; let left = [icc_3, two_spots, mask_8, past_limit]; @@ -614,6 +624,9 @@ fn images_whose_data_decodes_far_past_their_samples_are_left_alone() { assert_eq!(stream(&doc, packed).dict.int(b"BitsPerComponent"), Some(1), "8-bit black-and-white hex text in lines: packed"); } +/// Monochrome Images keeping each image's compression (and resampling as by default). +const RETAIN: MonoSettings = MonoSettings { compression: MonoCompression::Retain, ..MonoSettings::REDUCE }; + /// Eight-bit samples (0 or 255) from rows of one-bit ones. fn eight_bit(bits: &[u8], w: u32, h: u32) -> Vec { let stride = (w as usize).div_ceil(8); @@ -667,7 +680,7 @@ fn eight_bit_black_and_white_takes_the_smaller_of_one_bit_and_the_grey_settings( let (mut doc, img) = build(); optimize(&mut doc, &Settings::default()).unwrap(); let (mut retained, _) = build(); - optimize(&mut retained, &Settings { mono: MonoCompression::Retain, ..Settings::default() }).unwrap(); + optimize(&mut retained, &Settings { mono: RETAIN, ..Settings::default() }).unwrap(); let (s, kept) = (stream(&doc, img), stream(&retained, img)); let bits = if what == "dither" { 8 } else { 1 }; assert_eq!(s.dict.int(b"BitsPerComponent"), Some(bits), "{what}: {} bytes, {} with Retain", s.raw.len(), kept.raw.len()); @@ -757,7 +770,7 @@ fn eight_bit_black_and_white_images_become_one_bit() { let settings = Settings { gray: keep, ..Settings::default() }; let mut kept = doc.clone(); - let r = optimize(&mut kept, &Settings { mono: MonoCompression::Retain, ..settings.clone() }).unwrap(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..settings.clone() }).unwrap(); assert_eq!(r.images_recompressed, 0, "Retain keeps every image as it is"); let untouched = [shade, indexed, keyed].map(|r| raw(&doc, r)); @@ -775,6 +788,141 @@ fn eight_bit_black_and_white_images_become_one_bit() { assert!(render(&doc) == before, "the page looks the same"); } +/// Black-and-white images above 450 ppi are averaged down to 300 ppi, by default: 1-bit Flate, CCITT +/// Group 4 (alone and behind ASCIIHexDecode), an image mask (its inverting decode array stays) and +/// eight-bit grey of only black and white, all drawn at 600 ppi, are stored at half the size (as +/// Group 4, or 1-bit Flate when that is smaller, as it is on this clean text), each new pixel the +/// majority of the 2 × 2 it covers (a tie black). Left at their size: an image at 400 ppi (stored +/// again as Group 4), a Group 4 one at 400 ppi (not even decoded), one behind ASCIIHexDecode at +/// 400 ppi (stored again as Group 4 alone), and one with a soft mask. The page looks the same at +/// screen size. With `downsample` off nothing is resampled; with Retain, Flate stays Flate, CCITT +/// stays CCITT, and eight-bit grey is left to the grey settings. +#[test] +fn black_and_white_images_above_450_ppi_are_averaged_down_to_300() { + let mut doc = Document::new_empty(); + let (w, h) = (1200u32, 480u32); + let text = bilevel(w, h, false); + let grey = [("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]; + let g4 = |doc: &mut Document, entries: &[(&str, Object)]| { + let mut d = Dict::new(); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(w.into())); + d.set(b"Height".to_vec(), Object::Int(h.into())); + d.set(b"Filter".to_vec(), Object::Array(vec![Object::name("CCITTFaxDecode")])); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(w.into())); + d.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Dict(parms)])); + for (k, v) in entries { + d.set(k.as_bytes().to_vec(), v.clone()); + } + doc.add(Object::Stream(Stream::from_raw(d, pdfcraft_filters::encode_g4(&text, w, h).unwrap()))) + }; + // Group 4 behind ASCIIHexDecode, its parameters second in the array. + let behind_hex = |doc: &mut Document| { + let r = g4(doc, &grey); + let s = stream(doc, r); + let mut d = s.dict.clone(); + let parms = d.get(b"DecodeParms").and_then(Object::as_array).and_then(|a| a.first()).cloned().unwrap(); + d.set(b"Filter".to_vec(), Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("CCITTFaxDecode")])); + d.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, parms])); + let hex = s.raw.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); + doc.set(r, Object::Stream(Stream::from_raw(d, hex))); + r + }; + let flate = mono_image(&mut doc, w, h, &text, None, &grey); + let fax = g4(&mut doc, &grey); + let hex_fax = behind_hex(&mut doc); + let hex_at_400 = behind_hex(&mut doc); + let decode = Object::Array(vec![1.into(), 0.into()]); + let mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true)), ("Decode", decode.clone())]); + let mut d = Dict::new(); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(w.into())); + d.set(b"Height".to_vec(), Object::Int(h.into())); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let eight = doc.add(Object::Stream(Stream::flate(d, &eight_bit(&text, w, h)))); + let at_400 = mono_image(&mut doc, w, h, &text, None, &grey); + let fax_at_400 = g4(&mut doc, &grey); + let mut sm = Dict::new(); + sm.set(b"Subtype".to_vec(), Object::name("Image")); + sm.set(b"Width".to_vec(), Object::Int(2)); + sm.set(b"Height".to_vec(), Object::Int(1)); + sm.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + sm.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let smask = doc.add(Object::Stream(Stream::flate(sm, &[255, 200]))); + let soft = mono_image(&mut doc, w, h, &text, None, &[grey[0].clone(), grey[1].clone(), ("SMask", Object::Ref(smask))]); + // 144 × 57.6 pt is 2 × 0.8 in: 600 ppi. 216 × 86.4 pt is 400 ppi. + page( + &mut doc, + &[("A", flate), ("B", fax), ("C", mask), ("D", eight), ("E", at_400), ("F", fax_at_400), ("G", soft), ("H", hex_fax), ("I", hex_at_400)], + "q 144 0 0 57.6 20 700 cm /A Do Q q 144 0 0 57.6 200 700 cm /B Do Q q 0 0 1 rg 144 0 0 57.6 380 700 cm /C Do Q \ + q 144 0 0 57.6 20 600 cm /D Do Q q 216 0 0 86.4 200 580 cm /E Do Q q 216 0 0 86.4 20 480 cm /F Do Q \ + q 144 0 0 57.6 260 480 cm /G Do Q q 144 0 0 57.6 430 600 cm /H Do Q q 216 0 0 86.4 20 360 cm /I Do Q", + ); + let before = render(&doc); + let raw = |doc: &Document, r: ObjRef| stream(doc, r).raw; + let size = |doc: &Document, r: ObjRef| { + let d = stream(doc, r).dict; + (d.int(b"Width").unwrap(), d.int(b"Height").unwrap()) + }; + // Each new pixel: 1 when more than two of the 2 × 2 it covers are. + let stride = (w as usize).div_ceil(8); + let bit = |x: usize, y: usize| text[y * stride + x / 8] & (0x80 >> (x % 8)) != 0; + let mut halved = vec![0u8; (w as usize / 2).div_ceil(8) * h as usize / 2]; + for y in 0..h as usize / 2 { + for x in 0..w as usize / 2 { + let ones = [(0, 0), (1, 0), (0, 1), (1, 1)].iter().filter(|(dx, dy)| bit(2 * x + dx, 2 * y + dy)).count(); + if ones > 2 { + halved[y * (w as usize / 2).div_ceil(8) + x / 8] |= 0x80 >> (x % 8); + } + } + } + + // The grey settings left alone too: an eight-bit image would otherwise take them when smaller. + let mut off = doc.clone(); + let keep = ImageSettings { downsample: false, target_ppi: 150.0, above_ppi: 225.0, compression: Compression::Retain }; + let mono = MonoSettings { downsample: false, ..MonoSettings::REDUCE }; + let r = optimize(&mut off, &Settings { mono, gray: keep, ..Settings::default() }).unwrap(); + assert_eq!(r.images_resampled, 0, "{r:?}"); + assert!([flate, fax, mask, eight, hex_fax].iter().all(|i| size(&off, *i) == (1200, 480))); + + let mut kept = doc.clone(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..Settings::default() }).unwrap(); + assert_eq!(r.images_resampled, 5, "{r:?}"); + assert_eq!(stream(&kept, flate).dict.name(b"Filter"), Some(&b"FlateDecode"[..]), "Retain: Flate stays Flate"); + for i in [fax, hex_fax] { + assert_eq!(stream(&kept, i).dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "and CCITT stays CCITT: {i:?}"); + } + assert_eq!(raw(&kept, hex_at_400), raw(&doc, hex_at_400), "and nothing else changes"); + for i in [flate, fax, hex_fax] { + assert_eq!((size(&kept, i), kept.decoded_image(&stream(&kept, i)).unwrap()), ((600, 240), halved.clone()), "{i:?}"); + } + assert_eq!(stream(&kept, eight).dict.int(b"BitsPerComponent"), Some(8), "Retain leaves eight bits to the grey settings"); + + let untouched = [fax_at_400].map(|r| raw(&doc, r)); + let hex_len = raw(&doc, hex_at_400).len(); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!((r.images_resampled, r.images_recompressed), (5, 3), "{r:?}"); + for i in [flate, fax, mask, eight, hex_fax] { + let s = stream(&doc, i); + assert_eq!((size(&doc, i), s.dict.int(b"BitsPerComponent")), ((600, 240), if i == mask { None } else { Some(1) })); + assert!(matches!(s.dict.name(b"Filter"), Some(b"CCITTFaxDecode" | b"FlateDecode")), "{i:?}"); + assert_eq!(doc.decoded_image(&s).unwrap(), halved, "{i:?}"); + } + assert_eq!(stream(&doc, mask).dict.get(b"Decode"), Some(&decode), "the decode array stays"); + for i in [at_400, soft, hex_at_400] { + assert_eq!((size(&doc, i), stream(&doc, i).dict.name(b"Filter")), ((1200, 480), Some(&b"CCITTFaxDecode"[..])), "{i:?}"); + } + assert!(raw(&doc, hex_at_400).len() < hex_len, "the ASCIIHexDecode is dropped"); + assert_eq!(doc.decoded_image(&stream(&doc, hex_at_400)).unwrap(), text, "and the picture kept"); + assert_eq!([fax_at_400].map(|r| raw(&doc, r)), untouched); + let after = render(&doc); + let differ = before.iter().zip(&after).filter(|(a, b)| a.abs_diff(**b) > 64).count(); + assert!(differ * 200 < before.len(), "{differ} of {} channel values differ visibly", before.len()); +} + #[test] fn settings_choose_what_happens() { let mut doc = Document::new_empty(); diff --git a/crates/ui-egui/src/workflows/optimize_ui.rs b/crates/ui-egui/src/workflows/optimize_ui.rs index d3a5cfd4a1..d937592aba 100644 --- a/crates/ui-egui/src/workflows/optimize_ui.rs +++ b/crates/ui-egui/src/workflows/optimize_ui.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use egui::{Align, Layout, RichText}; -use pdfcraft_engine::optimize::{Compatibility, Compression, ImageSettings, MonoCompression, Preset, QUALITIES, Settings}; +use pdfcraft_engine::optimize::{Compatibility, Compression, ImageSettings, MonoCompression, MonoSettings, Preset, QUALITIES, Settings}; use pdfcraft_engine::optimizer::{OptimizeStage, Optimized}; use pdfcraft_engine::{DocId, EditError, Hidden, OptimizeReport}; @@ -143,18 +143,25 @@ fn image_row(ui: &mut egui::Ui, id: &str, title: &str, s: &mut ImageSettings) { ui.add_space(8.0); } -/// Monochrome Images: how 1-bit images and image masks are stored. -fn mono_row(ui: &mut egui::Ui, m: &mut MonoCompression) { +/// Monochrome Images: how black-and-white images and image masks are resampled and stored. +fn mono_row(ui: &mut egui::Ui, m: &mut MonoSettings) { ui.label(egui::RichText::new(tl!("Monochrome Images")).font(theme::semibold(13.0))); + ui.horizontal(|ui| { + ui.add(widgets::checkbox(&mut m.downsample, tl!("Downsample to"))); + ui.add_enabled(m.downsample, egui::DragValue::new(&mut m.target_ppi).range(9.0..=2400.0).suffix(" ppi")); + ui.label(tl!("for images above")); + ui.add_enabled(m.downsample, egui::DragValue::new(&mut m.above_ppi).range(9.0..=2400.0).suffix(" ppi")); + }); + m.above_ppi = m.above_ppi.max(m.target_ppi); ui.horizontal(|ui| { ui.label(tl!("Compression")); - let label = match m { + let label = match m.compression { MonoCompression::CcittG4 => tl!("CCITT Group 4"), MonoCompression::Retain => tl!("Retain existing"), }; craft_ui::ComboBox::from_id_salt(("mono", "compression")).selected_text(label).show_ui(ui, |ui| { - ui.selectable_value(m, MonoCompression::CcittG4, tl!("CCITT Group 4")); - ui.selectable_value(m, MonoCompression::Retain, tl!("Retain existing")); + ui.selectable_value(&mut m.compression, MonoCompression::CcittG4, tl!("CCITT Group 4")); + ui.selectable_value(&mut m.compression, MonoCompression::Retain, tl!("Retain existing")); }); }); ui.add_space(8.0); diff --git a/crates/ui-egui/tests/it/create.rs b/crates/ui-egui/tests/it/create.rs index 22ac8503e7..1c9a757651 100644 --- a/crates/ui-egui/tests/it/create.rs +++ b/crates/ui-egui/tests/it/create.rs @@ -188,7 +188,11 @@ fn the_pdf_optimizer_dialog_saves_an_optimized_copy() { h.get_by_label("PDF Optimizer"); h.get_by_label("Color Images"); h.get_by_label("Monochrome Images"); - assert_eq!(h.state().optimize_draft.settings.mono, pdfcraft_engine::optimize::MonoCompression::CcittG4); + assert_eq!(h.state().optimize_draft.settings.mono, pdfcraft_engine::optimize::MonoSettings::REDUCE); + // Monochrome Images ▸ Downsample to can be turned off. + h.get_by_label("Downsample to").click(); + h.run_steps(2); + assert!(!h.state().optimize_draft.settings.mono.downsample); h.get_by_label("Discard Objects").click(); h.run_steps(2); h.get_by_label("Discard document tags").click(); diff --git a/docs/pdf-spec-parity.md b/docs/pdf-spec-parity.md index dba04f6c4c..6ed9eba7c1 100644 --- a/docs/pdf-spec-parity.md +++ b/docs/pdf-spec-parity.md @@ -1,6 +1,6 @@ # PDF specification and standards parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (§7.4: CCITT Group 4 encoding in the Optimizer; earlier, §14.3: custom properties edited in Document Properties) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (§7.4: CCITT decoding in `pdfcraft-filters`; earlier, §7.4: CCITT Group 4 encoding in the Optimizer) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) How much of ISO 32000-2 (PDF 2.0) and the PDF subset standards PdfCraft implements, set against what Acrobat Pro implements. This is PdfCraft's equivalent of a "geometry" or "codec" checklist: @@ -21,7 +21,7 @@ are borrowed; PDF 2.0 additions and most subset standards are missing. **150–2 |---|---|---|---|---| | §7.2–7.5 | Syntax, objects, file structure, xref tables and streams, object streams, hybrid files, incremental updates | full | **own**, tolerant, repairs damaged files | `crates/cos`; corpus 958 files; fuzzed nightly | | §7.4 | Stream filters: Flate, LZW, ASCII85, ASCIIHex, RunLength, predictors | full | **own** | `crates/filters` | -| §7.4 | Image filters: DCT, JPX, JBIG2, CCITTFax | full | **borrowed** (decode); CCITT Group 4 encoding | `core.image-filters` partial; own codecs M1.1/M1.2; the Optimizer encodes CCITT Group 4 (`pdfcraft_filters::encode_g4`, `optimize.mono-compression`); no JBIG2 encoding | +| §7.4 | Image filters: DCT, JPX, JBIG2, CCITTFax | full | **borrowed** (decode); CCITT Group 4 encoding | `core.image-filters` partial; own codecs M1.1/M1.2; CCITT decoding in `pdfcraft-filters` (`decode_ccitt`, through `hayro-ccitt`) for export and the Optimizer, which encodes Group 4 (`optimize.mono-compression`); no JBIG2 encoding | | §7.5.8 | Linearization | R W | R only | `core.linearization` planned | | §7.6.4 | Standard security handler R2–R6 (RC4 40/128, AES-128, AES-256), crypt filters, `/EncryptMetadata`, SASLprep | full | **own**, read and write | `crates/crypt`; written files open in Acrobat DC since #836 (#774: Acrobat needs the catalog outside object streams) | | §7.6.5 | Public-key security handler | full | none | `core.open-public-key`, `protect.certificate-encryption` planned | @@ -130,6 +130,7 @@ are borrowed; PDF 2.0 additions and most subset standards are missing. **150–2 | Date | Change | Summary | |---|---|---| +| 2026-10-26 | minor | §7.4: CCITT decoding (`hayro-ccitt`) in `pdfcraft-filters`, for export and the Optimizer | | 2026-10-26 | minor | §7.4: CCITT Group 4 encoding (`fax`) in the Optimizer | | 2026-10-14 | minor | §12.4.4: `/Trans` and `/Dur` read into `DocInfo::transitions` and reported by `doc_info` | | 2026-10-11 | minor | §7.6.4: encrypted files we write open in Acrobat (#774, fixed by #836) | diff --git a/docs/roadmap.md b/docs/roadmap.md index 3e4c971b2f..6d12e87564 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,7 +2,7 @@ > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) -> **Last reviewed:** 2026-10-24 · **Last updated:** 2026-10-24 · **Change:** minor (M11: black-and-white pictures stay one bit when PDFs are created and optimized; CCITT images export; estimates unchanged) +> **Last reviewed:** 2026-10-24 · **Last updated:** 2026-10-24 · **Change:** minor (M11: black-and-white images above 450 ppi are averaged down to 300 ppi by the optimizer; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M11: the PDF Optimizer and Reduce File Size store black-and-white images as CCITT Group 4 when smaller; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M11: image optimization accounts for page-local UserUnit; estimates unchanged) @@ -257,6 +257,7 @@ Estimates unchanged. | Date | Change | Summary | |---|---|---| +| 2026-10-24 | minor | M11: monochrome downsampling: the PDF Optimizer and Reduce File Size average black-and-white images (1-bit, CCITT fax, eight-bit black-and-white grey) above 450 ppi down to 300 ppi, by area with ties black, as Monochrome Images ▸ Downsample to and `doc_optimize` `mono: { downsample, ppi, above_ppi }`; CCITT decoding moves into `pdfcraft-filters` (`decode_ccitt`) behind `Document::decoded_image`. Estimates unchanged. | | 2026-10-24 | minor | M11: Create PDF stores pictures of only black and white at one bit (Group 4 or Flate, whichever is smaller) instead of eight-bit grey; the optimizer packs eight-bit black-and-white grey images to one bit before Monochrome Images compresses them, unless the grey settings make them smaller, as they can a dithered scan, soft mask counted; Export ▸ Images decodes CCITT fax images. Estimates unchanged. | | 2026-10-23 | minor | M11: black-and-white images (1-bit, and image masks) are stored losslessly as CCITT Group 4 when that is smaller: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size too), `doc_optimize` `mono`; `pdfcraft_filters::encode_g4`. JBIG2 and monochrome downsampling not yet. Estimates unchanged. | | 2026-10-27 | minor | M10: OCR stages SHA-256-verified ONNX models after the upstream RTEN URL brownout; discovery prefers a complete ONNX pair per directory, falls back to a complete RTEN pair, and never mixes formats or directories; language/web limits and estimates unchanged. | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 376131a974..9a8597e520 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -130,8 +130,8 @@ milestone = "M1" status = "partial" commands = [] tools = [] -evidence = [] -notes = "decoded through the hayro bootstrap for display; own codecs are M1.1/M1.2" +evidence = ["crates/filters/src/ccitt.rs::decoding_gives_the_rows_back", "crates/cos/src/document/tests.rs::fax_images_decode_with_their_parameters"] +notes = "decoded through the hayro bootstrap for display; own codecs are M1.1/M1.2. CCITT fax decoding is in filters (decode_ccitt, through hayro-ccitt), which Document::decoded_image uses for Export ▸ Images and the optimizer" [[feature]] id = "core.decompression-limits" @@ -7360,8 +7360,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced"] tools = ["doc_optimize"] -evidence = ["crates/optimize/tests/optimize.rs::images_are_measured_where_drawn_and_downsampled", "crates/optimize/tests/optimize.rs::settings_choose_what_happens", "crates/automation/tests/it/automation.rs::optimizing_through_tools", "crates/optimize/tests/optimize.rs::a_page_picture_printed_as_one_pixel_strips_is_merged_into_one_image", "crates/optimize/tests/optimize.rs::images_that_do_not_tile_a_picture_are_left_alone", "crates/optimize/tests/optimize.rs::a_deeply_nested_form_is_measured_at_its_largest_use", "crates/optimize/tests/optimize.rs::images_are_not_resampled_when_some_uses_could_not_be_measured", "crates/optimize/tests/optimize.rs::image_resolution_uses_the_pages_physical_unit", "crates/optimize/tests/optimize.rs::optimization_retains_an_image_below_threshold_on_a_scaled_page", "crates/optimize/tests/optimize.rs::shared_forms_are_measured_in_each_pages_physical_units"] -notes = "colour and grayscale, bicubic, effective resolution measured where drawn (every use, with page-local UserUnit scaling, through nested forms up to 12 deep; when a use can't be measured, nothing is resampled); image fragments stacked edge to edge are merged first (side-by-side tiles not yet); monochrome not yet" +evidence = ["crates/optimize/tests/optimize.rs::images_are_measured_where_drawn_and_downsampled", "crates/optimize/tests/optimize.rs::settings_choose_what_happens", "crates/automation/tests/it/automation.rs::optimizing_through_tools", "crates/optimize/tests/optimize.rs::a_page_picture_printed_as_one_pixel_strips_is_merged_into_one_image", "crates/optimize/tests/optimize.rs::images_that_do_not_tile_a_picture_are_left_alone", "crates/optimize/tests/optimize.rs::a_deeply_nested_form_is_measured_at_its_largest_use", "crates/optimize/tests/optimize.rs::images_are_not_resampled_when_some_uses_could_not_be_measured", "crates/optimize/tests/optimize.rs::image_resolution_uses_the_pages_physical_unit", "crates/optimize/tests/optimize.rs::optimization_retains_an_image_below_threshold_on_a_scaled_page", "crates/optimize/tests/optimize.rs::shared_forms_are_measured_in_each_pages_physical_units", "crates/optimize/tests/optimize.rs::black_and_white_images_above_450_ppi_are_averaged_down_to_300", "crates/optimize/src/images.rs::it_matches_the_slow_way", "crates/automation/tests/it/automation.rs::doc_optimize_averages_black_and_white_scans_down", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy"] +notes = "colour and grayscale, bicubic, effective resolution measured where drawn (every use, with page-local UserUnit scaling, through nested forms up to 12 deep; when a use can't be measured, nothing is resampled); image fragments stacked edge to edge are merged first (side-by-side tiles not yet); monochrome (1-bit, CCITT fax and eight-bit black-and-white grey) averaged by area to 300 ppi above 450 ppi by default, a tie black, then stored as Group 4 or 1-bit Flate (Monochrome Images ▸ Downsample to; doc_optimize mono); not monochrome images with a soft mask, and Acrobat's bicubic and subsampling choices for monochrome aren't offered" [[feature]] id = "optimize.recompress-images" @@ -7385,7 +7385,7 @@ status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] evidence = ["crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy"] -notes = "CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. An eight-bit grey image of only black and white is packed to one bit first, then stored the same way (indexed images and colour-key masks aside), unless the grey settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Not yet: JBIG2 (lossless or lossy, which Adobe's help says beats CCITT for monochrome), monochrome downsampling, inline images, soft masks and /Mask images, rows wider than 32,767 samples. Redaction doesn't decode Group 4 yet (a mark over part of such an image removes it whole; nothing leaks)." +notes = "CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. An eight-bit grey image of only black and white is packed to one bit first, then stored the same way (indexed images and colour-key masks aside), unless the grey settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Not yet: JBIG2 (lossless or lossy, which Adobe's help says beats CCITT for monochrome), Group 3 to Group 4 without resampling, inline images, soft masks and /Mask images, rows wider than 32,767 samples. Redaction doesn't decode Group 4 yet (a mark over part of such an image removes it whole; nothing leaks)." [[feature]] id = "optimize.only-if-smaller"