diff --git a/crates/filters/README.md b/crates/filters/README.md index 0113e22179..5df0761eb9 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -54,7 +54,7 @@ encode_jbig2_generic(data, columns, rows) -> Result, FilterError> // t - `ccitt.rs`: Group 4 output decoded by `hayro-ccitt` (dev-dependency, an independent decoder) matches the input bits: a text-like page, padding bits, the widest row, and property tests over bitmaps of noise and of runs; any input returns rather than panics. - `mq.rs`: the arithmetic coder reproduces Annex H.2's 30 bytes from its 256 decisions. - `jbig2.rs`: output decoded by `hayro-jbig2` (dev-dependency, an independent decoder) matches the input bits: a text-like page (smaller than Group 4), edges, padding, all-white and all-black regions (typical rows), and property tests over bitmaps of noise and of runs; any input returns rather than panics. -- `jbig2/symbols.rs`: the integer coders against Annex A's example and Table A.1; symbol-coded output decoded by `hayro-jbig2` matches the input bits for every plan and strip size: text whose letters repeat (under half the generic coding's size, the dictionary retained and referred to), large and lone marks left to a generic region, marks on the edges, touching at corners and inside each other's boxes, and property tests over glyphs scattered over each other and the edges with stray pixels (never larger than the generic coding); one shape, a blank page and noise are coded generically; any input returns rather than panics. +- `jbig2/symbols.rs`: the integer coders against Annex A's example and Table A.1; symbol-coded output decoded by `hayro-jbig2` matches the input bits for every plan and strip size: text whose letters repeat (under half the generic coding's size, the dictionary retained and referred to), large and lone marks left to a generic region, marks on the edges, touching at corners and inside each other's boxes, and property tests over glyphs scattered over each other and the edges with stray pixels (never larger than the generic coding); one shape, a blank page and noise are coded generically; marks joined at a corner are one mark, and 256 pixels a side is a symbol while 257 is left out; black padding bits are ignored; coordinates past 4,435 use Annex A's 32-bit range and PREV stays below 512; past 2^20 runs or 2^24 rows the image is coded generically; any input returns rather than panics. - `fuzz/`: a detached cargo-fuzz crate (`cargo +nightly fuzz run decode`, run from `crates/filters/`). ## Status diff --git a/crates/filters/src/jbig2/symbols.rs b/crates/filters/src/jbig2/symbols.rs index 00aa187602..48013e9864 100644 --- a/crates/filters/src/jbig2/symbols.rs +++ b/crates/filters/src/jbig2/symbols.rs @@ -655,9 +655,97 @@ mod tests { let big = bits(Some(4436)); assert_eq!((big.len(), &big[..6]), (6 + 32, &[0, 1, 1, 1, 1, 1][..])); assert!(big[6..].iter().all(|b| *b == 0)); - // PREV keeps a leading 1 and the last eight decisions. - let contexts: Vec = decisions(Some(i32::MIN)).iter().map(|(cx, _)| *cx).collect(); - assert!(contexts[..8].iter().all(|cx| *cx < 256) && contexts[8..].iter().all(|cx| (256..512).contains(cx))); + // PREV keeps a leading 1 and the last eight decisions, so it stays below 512, also when + // those eight are all 0 (PREV exactly 256), as in 4436's offset of 0. + for value in [Some(i32::MIN), Some(4436), Some(-4436), Some(4436 + 0x100), Some(i32::MAX), None] { + let contexts: Vec = decisions(value).iter().map(|(cx, _)| *cx).collect(); + assert!(contexts.iter().take(8).all(|cx| *cx < 256) && contexts.iter().skip(8).all(|cx| (256..512).contains(cx)), "{value:?}"); + } + assert!(decisions(Some(4436)).iter().any(|(cx, _)| *cx == 256)); + } + + /// Strips and first columns thousands of pixels apart need Annex A's 32-bit range: a mark at + /// the far right of a 12,000-pixel page, then one at the left below it (S back by 11,000), and + /// lines 5,000 rows apart. Each shape occurs twice, so both plans are made. + #[test] + fn coordinates_past_4435_round_trip() { + let shapes = glyphs(3, 4); + let mut picture = vec![vec![0u8; 12_000]; 40]; + for (k, x) in [11_000, 0, 5_000, 11_980].into_iter().enumerate() { + draw(&mut picture, &shapes[k % 2], x, 2 + 9 * k as i64); + } + assert_eq!(every_plan_round_trips(&picture), 8); + let mut tall = vec![vec![0u8; 30]; 10_050]; + for (k, y) in [0, 5_000, 10_030].into_iter().enumerate() { + draw(&mut tall, &shapes[k % 3], 3, y); + draw(&mut tall, &shapes[(k + 1) % 3], 15, y); + } + assert_eq!(every_plan_round_trips(&tall), 8); + } + + /// The padding bits of a row whose width isn't a multiple of 8 are ignored whatever they are: + /// black (0) padding, as many scans have, neither joins the marks nor stops symbol coding. + #[test] + fn black_padding_bits_are_ignored() { + for columns in [13, 241, 901] { + let picture = text(columns, 120, &glyphs(12, 6), 8); + let mut data = samples(&picture); + let stride = columns.div_ceil(8); + for row in data.chunks_mut(stride) { + row[stride - 1] &= 0xFF << (8 * stride - columns); + } + let (c, r) = (columns as u32, 120); + let page = page(&data, c, r).unwrap(); + assert!(page.marks.iter().all(|m| m.x1 as usize <= columns), "{columns} columns"); + for copies in [1, 2] { + if let Some(coded) = plan(&page, c, r, copies, None) { + assert_eq!(decode(&coded), unpack(&data, c, r), "{columns} columns"); + } + } + if columns > 13 { + assert_eq!(segment_types(&encode_jbig2(&data, c, r).unwrap())[1], 0, "{columns} columns: symbol coded"); + } + } + } + + /// Pixels touching only at a corner are one mark (8-connected); side by side with a white + /// column between, two. A mark of 256 pixels a side is a symbol, one of 257 is left to the + /// generic region. The diagonal and the dot occur twice, so both plans are made. + #[test] + fn marks_join_at_corners_and_large_ones_are_left_out() { + let mut picture = vec![vec![0u8; 300]; 300]; + for i in 0..6 { + picture[10 + i][10 + i] = 1; + picture[10 + i][100 + i] = 1; + } + picture[30][10] = 1; + picture[30][12] = 1; + picture[40][0..256].fill(1); + picture[60][0..257].fill(1); + for row in &mut picture[44..300] { + row[290] = 1; + } + let data = samples(&picture); + let page = page(&data, 300, 300).unwrap(); + let boxes: Vec<(u32, u32, u32, u32)> = page.marks.iter().map(|m| (m.x0, m.y0, m.x1 - m.x0, m.y1 - m.y0)).collect(); + assert_eq!(boxes, [(10, 10, 6, 6), (100, 10, 6, 6), (10, 30, 1, 1), (12, 30, 1, 1), (0, 40, 256, 1), (290, 44, 1, 256), (0, 60, 257, 1)]); + let large: Vec<(u32, u32)> = page.large.iter().map(|m| (page.marks[*m as usize].x0, page.marks[*m as usize].y0)).collect(); + assert_eq!(large, [(0, 60)]); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Past 2^20 black runs or 2^24 rows the image is left to generic coding; at 2^20 runs it is + /// still looked at. + #[test] + fn busy_or_tall_images_are_left_to_generic_coding() { + // 0x55: four black pixels a byte, each its own run, under the same ones on the next row. + let columns = 1 << 20; + let row = vec![0x55u8; columns / 8]; + assert!(page(&row.repeat(2), columns as u32, 2).is_some(), "2^20 runs"); + assert!(page(&row.repeat(3), columns as u32, 3).is_none(), "1.5 × 2^20 runs"); + let tall = vec![0xFFu8; MAX_ROWS as usize + 1]; + assert!(page(&tall, 1, MAX_ROWS + 1).is_none()); + assert!(page(&tall, 1, MAX_ROWS).is_some()); } /// A page of text whose letters repeat exactly, as rendered text does, keeps each letter once: diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 4c8ae1cd3e..97915f8d8c 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -40,10 +40,12 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going or lossless JBIG2 (`encode_jbig2`; a document below PDF 1.4 gets `/Version /1.4` in its catalog) when that is smaller as stored. JBIG2 stores each distinct mark once, as a symbol, when the image's marks repeat exactly, as rendered text's do: on six pages of - 300 dpi text that came to 28% of a generic region's size and 17% of Group 4's. On scans, whose copies of a letter are seldom identical, it is a generic region, - about a third smaller than Group 4. It is lossless: the samples decode to the same bits, so the colour - space, decode array and masks stay. Group 4 wins on scans, whose ragged edges defeat Flate; - on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Not yet: + 300 dpi text that came to 28% of a generic region's size and 17% of Group 4's. On scans, + whose copies of a letter are seldom identical, it is nearly always a generic region: 27–33% + smaller than Group 4 on the same pages with scan-like noise added. It is lossless: the + samples decode to the same bits, so the colour space, decode array and masks stay. Group 4 + wins on scans, whose ragged edges defeat Flate; on clean computer-drawn 1-bit art Flate can + be smaller, and the image then stays. Not yet: JBIG2 refinement coding (lossless gains on scans), a symbol dictionary shared between images (`/JBIG2Globals`), lossy JBIG2, monochrome downsampling, inline images, soft masks and `/Mask` images, and Group 4 for rows wider than 32,767 samples (JBIG2 takes rows of up to diff --git a/docs/roadmap.md b/docs/roadmap.md index 804af97178..f21943c38f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,8 +1,8 @@ # PdfCraft roadmap: milestones and what's next > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) -> **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-26 · **Last updated:** 2026-10-26 · **Change:** minor (M11: lossless JBIG2 stores each distinct mark once, as a symbol; estimates unchanged) +> **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-24 · **Change:** minor (M11: the PDF Optimizer can store black-and-white images as lossless JBIG2, and Export ▸ Images reads JBIG2 images; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M11: the PDF Optimizer and Reduce File Size store black-and-white images as CCITT Group 4 when smaller; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index c18856e3aa..e801495fde 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7385,7 +7385,7 @@ status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] evidence = ["crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy"] -notes = "CCITT Group 4 (the default, so Reduce File Size does it too) or JBIG2, both lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. JBIG2 stores each distinct mark once in a symbol dictionary and places its exact copies with a text region, larger and lone marks in a generic region (template 0, typical prediction), or the whole image as a generic region, whichever is smaller: on rendered 300 dpi text 28% of the generic region's size and 17% of Group 4's; on scans, whose copies of a letter are seldom identical, the generic region, about a third smaller than Group 4. A document below PDF 1.4 gets /Version 1.4. Not yet: JBIG2 refinement coding of near-copies (where lossless gains on scans would come from), a symbol dictionary shared between images (/JBIG2Globals), JBIG2's lossy mode, monochrome downsampling, inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples (JBIG2 takes rows of up to 1,048,576). Redaction doesn't decode JBIG2 or Group 4 yet: a mark over part of such an image removes the whole image (nothing leaks). Export reads JBIG2, but nothing that exports an image reads Group 4 yet: Export ▸ Images and Save Image As refuse it, Copy Image fails, and Word, PowerPoint and HTML export leave the picture out." +notes = "CCITT Group 4 (the default, so Reduce File Size does it too) or JBIG2, both lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. JBIG2 stores each distinct mark once in a symbol dictionary and places its exact copies with a text region, larger and lone marks in a generic region (template 0, typical prediction), or the whole image as a generic region, whichever is smaller: on rendered 300 dpi text 28% of the generic region's size and 17% of Group 4's; on scans, whose copies of a letter are seldom identical, nearly always the generic region (27–33% smaller than Group 4 on the same pages with scan-like noise added). A document below PDF 1.4 gets /Version 1.4. Not yet: JBIG2 refinement coding of near-copies (where lossless gains on scans would come from), a symbol dictionary shared between images (/JBIG2Globals), JBIG2's lossy mode, monochrome downsampling, inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples (JBIG2 takes rows of up to 1,048,576). Redaction doesn't decode JBIG2 or Group 4 yet: a mark over part of such an image removes the whole image (nothing leaks). Export reads JBIG2, but nothing that exports an image reads Group 4 yet: Export ▸ Images and Save Image As refuse it, Copy Image fails, and Word, PowerPoint and HTML export leave the picture out." [[feature]] id = "optimize.only-if-smaller"