From 25eb568839f97bc50c97297f9f474b1e3c3a2d22 Mon Sep 17 00:00:00 2001 From: Ahmad Mujahid Date: Tue, 20 Oct 2026 13:30:02 +0400 Subject: [PATCH] fix(forms): preserve content stream comment boundaries (#3502) --- ROADMAP.md | 4 ++ crates/automation/tests/automation.rs | 59 +++++++++++++++++++++++++++ crates/forms/src/detect.rs | 9 ++-- crates/forms/src/tests.rs | 32 +++++++++++++++ docs/roadmap.md | 4 +- 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index b1283e775..ad251d21f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,6 +2,7 @@ **Stage: alpha** · next: beta, ~25% (ready for real work 50% → 75%) and ~400–700 h away +> **Last reviewed:** 2026-10-20 - **Last updated:** 2026-10-20 - **Change:** minor (M6, #3502: form detection uses the existing content-stream boundary policy; estimates unchanged) > **Last reviewed:** 2026-10-18 · **Last updated:** 2026-10-18 · **Change:** minor (M5, PR #1971: bounded nested FDF export excludes source page/page-tree dictionaries; refreshed against upstream 4db6ac4a; estimates unchanged) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M7: the Edit text caret is as tall as the row it is on; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M6: CSV tab-containing headers; estimates unchanged) @@ -172,6 +173,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-20 (M6, #3502):** Form detection preserves comments across content streams and does not add fields over rectangles hidden by those comments. Synthetic tests cover visible controls, graphics state, and saved field inventories. Estimates unchanged. + - **2026-10-19 (M14, #3373):** The main menu and its long submenus scroll in short windows, keeping their commands reachable. Estimates unchanged. - **2026-10-19 (M3, #3387):** Save-prompt shortcuts yield to later consent and utility dialogs; a synthetic UI regression checks that an edited document stays open. Estimates unchanged. @@ -552,6 +555,7 @@ Newest first: the date, what moved, and the new overall estimate. | Date | Change | Summary | |---|---|---| +| 2026-10-20 | minor | M6, #3502: form detection preserves comments across content streams; estimates unchanged. | | 2026-10-18 | minor | M5, PR #1971: nested source `/Page` and `/Pages` dictionaries export as null without traversing page content or the page tree. Synthetic saved/reopened three-page regression retains unrelated metadata and aliases. FDF import semantics, copied-graph limits and estimates unchanged. | | 2026-10-15 | minor | M5: budget retained FDF entries before cloning; discovery keeps lazy array owners and annotation positions, with alias/cache-eviction stress fixtures. Limits cover the copied graph; source parsing/page discovery retain existing COS/parser limits, with no total source-memory bound. FDF import semantics and estimates unchanged. | | 2026-10-15 | minor | M7: the Edit text caret is as tall as the row it is on in a region with several sizes; estimates unchanged. | diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index cbf40f196..4068e6b8d 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -5126,6 +5126,65 @@ fn detecting_form_fields_through_tools() { assert_eq!(f["fields"][1]["value"], "Lisbon", "{f}"); } +#[test] +fn detecting_form_fields_preserves_content_piece_comments() { + for visible in [false, true] { + let first = format!("BT /F1 18 Tf 40 250 Td (FIRST) Tj ET % trailing comment{}", if visible { "\n" } else { "" }); + let last = b"40 120 300 30 re S\nBT /F1 18 Tf 40 50 Td (LAST) Tj ET\n"; + let mut source = pdfcraft_cos::Document::open(std::sync::Arc::new(fixture(1))).unwrap(); + let page = pdfcraft_model::pages(&source)[0].obj; + let contents = [first.as_bytes(), last] + .iter() + .map(|bytes| pdfcraft_cos::Object::Ref(source.add(pdfcraft_cos::Object::Stream(pdfcraft_cos::Stream::flate(Default::default(), bytes))))) + .collect(); + source + .update_dict(page, |d| { + d.set(b"Contents".to_vec(), pdfcraft_cos::Object::Array(contents)); + d.set(b"MediaBox".to_vec(), pdfcraft_cos::Object::Array(vec![0.into(), 0.into(), 400.into(), 300.into()])); + }) + .unwrap(); + let dir = workdir(if visible { "detect-visible-piece" } else { "detect-hidden-piece" }); + std::fs::write(dir.join("pieces.pdf"), pdfcraft_cos::write_full(&source, &Default::default()).unwrap()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({"path": "pieces.pdf"}))["doc"].as_u64().unwrap(); + let candidates = if visible { json!([{"page": 1, "kind": "text", "name": "Text1", "rect": [41.0, 121.0, 339.0, 149.0]}]) } else { json!([]) }; + let inventory = if visible { + json!([{"name": "Text1", "type": "text", "value": null, "page": 1, "rect": [41.0, 151.0, 339.0, 179.0], "rotation": 0, "read_only": false, "required": false, "multiline": false}]) + } else { + json!([]) + }; + let check_text = |a: &mut Automation, doc| { + let text = page_text(a, doc).join(" "); + assert_eq!(text.split_whitespace().collect::>(), ["FIRST", "LAST"]); + }; + let check_render = |a: &mut Automation, doc| { + let (width, height, pixels) = a.render_pixels(doc, 1, 1.0).unwrap(); + assert_eq!((width, height), (400, 300)); + let region: Vec<_> = (145..185).flat_map(|y| (35..345).map(move |x| (y * 400 + x) * 4)).collect(); + if visible { + assert!(region.iter().any(|&i| pixels[i..i + 3].iter().all(|&v| v < 128)), "the visible box has ink"); + } else { + assert!(region.iter().all(|&i| pixels[i..i + 3] == [255, 255, 255]), "the hidden box region stays white"); + } + }; + check_text(&mut a, doc); + check_render(&mut a, doc); + assert_eq!(ok(&mut a, "form_fields", json!({"doc": doc}))["fields"], json!([])); + assert_eq!(ok(&mut a, "form_detect_fields", json!({"doc": doc, "add": false}))["fields"], candidates); + assert_eq!(ok(&mut a, "form_fields", json!({"doc": doc}))["fields"], json!([])); + assert_eq!(ok(&mut a, "form_detect_fields", json!({"doc": doc}))["fields"], candidates); + assert_eq!(ok(&mut a, "form_fields", json!({"doc": doc}))["fields"], inventory); + assert_eq!(ok(&mut a, "form_detect_fields", json!({"doc": doc}))["fields"], json!([])); + ok(&mut a, "doc_save", json!({"doc": doc, "path": "saved.pdf"})); + let mut reopened = auto(&dir); + let again = ok(&mut reopened, "doc_open", json!({"path": "saved.pdf"}))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut reopened, "form_fields", json!({"doc": again}))["fields"], inventory); + check_text(&mut reopened, again); + check_render(&mut reopened, again); + std::fs::remove_dir_all(dir).unwrap(); + } +} + #[test] fn detecting_comb_boxes_through_tools() { let mut body = String::from("BT /F1 10 Tf 40 105 Td (Postcode) Tj ET\n"); diff --git a/crates/forms/src/detect.rs b/crates/forms/src/detect.rs index d5bf073b1..8c980c09f 100644 --- a/crates/forms/src/detect.rs +++ b/crates/forms/src/detect.rs @@ -5,7 +5,7 @@ //! [`detect`] is a pure function over the page's words and drawn shapes; [`page_shapes`] reads //! the shapes from a page's content, [`raster::shapes`] from a picture of the page (a scan). -use pdfcraft_content::{Matrix, parse}; +use pdfcraft_content::{Matrix, Pieces}; use pdfcraft_cos::{Document, Object}; pub mod raster; @@ -136,11 +136,10 @@ pub fn page_shapes(doc: &Document, page: usize) -> Shapes { }, None => Vec::new(), }; - let mut data = Vec::new(); + let mut pieces = Vec::new(); for o in list { if let Object::Stream(s) = &*doc.resolve(&o) { - data.extend(s.decoded().unwrap_or_default()); - data.push(b'\n'); + pieces.push(s.decoded().unwrap_or_default()); } } let mut ctm = Matrix::IDENTITY; @@ -149,7 +148,7 @@ pub fn page_shapes(doc: &Document, page: usize) -> Shapes { let mut subpaths: Vec = Vec::new(); let mut path = DetectionPath::default(); let mut cur: Option<[f64; 2]> = None; - for op in parse(&data).ops { + for op in Pieces::join(&pieces).parse().ops { match op.op.as_slice() { b"q" => stack.push(ctm), b"Q" => ctm = stack.pop().unwrap_or(ctm), diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index fef5bcfbd..ad0114009 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -1575,6 +1575,38 @@ fn page_shapes_reads_boxes_and_rules() { assert_eq!(sh.rules, [[40.0, 100.0, 240.0, 100.0], [60.0, 600.5, 260.0, 600.5]]); } +#[test] +fn page_shapes_preserves_content_piece_boundaries() { + use crate::detect::{Candidate, Kind, Shapes, detect, page_shapes}; + let read = |pieces: &[&[u8]]| { + let mut doc = one_page(); + let page = pdfcraft_model::pages(&doc)[0].obj; + let contents = + pieces.iter().map(|bytes| Object::Ref(doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Default::default(), bytes))))).collect(); + doc.update_dict(page, |d| d.set(b"Contents".to_vec(), Object::Array(contents))).unwrap(); + page_shapes(&doc, 0) + }; + let empty = Shapes::default(); + for pieces in [vec![b"% trailing".as_slice(), b"40 120 300 30 re S\n"], vec![b"% trailing".as_slice(), b"", b"", b"40 120 300 30 re S\n"]] { + let shapes = read(&pieces); + assert_eq!(shapes, empty, "an unterminated comment hides the next piece's first line"); + assert_eq!(detect(&[], &shapes, &[], &[]), Vec::::new()); + } + let visible = Shapes { boxes: vec![[40.0, 120.0, 340.0, 150.0]], rules: vec![] }; + for ending in [b"% trailing\n".as_slice(), b"% trailing\r", b"% trailing\r\n"] { + let shapes = read(&[ending, b"40 120 300 30 re S\n"]); + assert_eq!(shapes, visible); + assert_eq!(detect(&[], &shapes, &[], &[]), vec![Candidate { kind: Kind::Text, rect: [41.0, 121.0, 339.0, 149.0], name: "Text1".into() }]); + } + assert_eq!(read(&[b"40 120", b"", b"300 30", b"re", b"S"]), visible); + assert_eq!( + read(&[b"q 2 0 0 2 0 0 cm 10 10", b"60 20 re", b"S Q 10 20 m 100 20 l S"]), + Shapes { boxes: vec![[20.0, 20.0, 140.0, 60.0]], rules: vec![[10.0, 20.0, 100.0, 20.0]] } + ); + assert_eq!(read(&[]), empty); + assert_eq!(read(&[b""]), empty); +} + fn close(a: f64, b: f64) -> bool { (a - b).abs() < 0.05 } diff --git a/docs/roadmap.md b/docs/roadmap.md index 5d73511e9..fd61c7e02 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,6 +2,7 @@ > **Last reviewed:** 2026-10-18 · **Last updated:** 2026-10-18 · **Change:** minor (M5, PR #1971: bounded nested FDF export excludes source page/page-tree dictionaries; refreshed against upstream 4db6ac4a; estimates unchanged) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M14/#2383 GPU-limit candidate; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-20 - **Last updated:** 2026-10-20 - **Change:** minor (M6, #3502: form detection uses the existing content-stream boundary policy; estimates unchanged) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M7: the Edit text caret is as tall as the row it is on; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M6: CSV tab-containing headers; estimates unchanged) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-18 · **Change:** minor (M4, #1919: repeated page indices accumulate rotations; M4: indirect page rotations refreshed upstream; M13: CLI check visits each directory once through aliases and cycles; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -133,7 +134,7 @@ total matches the parity estimate. | M4 | Engine, history, save, organize | 100–180 | 95% | 8–15 | Done:
- command registry (menus, shortcuts and palette all use it);
- undo/redo; incremental, atomic and encrypted saves;
- autosave and crash recovery;
- organize, combine, extract, split and insert-from-file, with identical fonts and images stored once;
- CLI `edit/combine/extract/split`.
Done since: Combine files in its own tab (sortable, resizable columns; folders; early warnings; unlocking protected files), bookmark editing, page labels (Number pages), CLI `run`, Set Page Boxes, Crop tool, Duplicate pages, Ctrl+A / Cmd+A to select all pages in Organize, Replace pages, bookmarks from tagged headings. Missing: page transitions, recovery on the web M4: Insert-from-file preserves source bookmarks; integration tests retained with current main. | | M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: stamps placed on a rotated page read upright (#1398); polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles. Missing: custom stamps, replace-text proposals, summary layouts with the page | | M5 | Comments (all annotation types, XFDF) | 120–200 | 88% | 12–25 | Done: notes, highlight/underline/strikeout/squiggly, text boxes, ink, lines, arrows, rectangles, ovals, stamps (dynamic, Sign Here, Standard Business), with appearance streams; replies, status, checkmarks, locking, move/resize/restyle/delete; properties dialog; panel filter and sort; hide all; summaries (comments only); XFDF/FDF import and export (partial imports preserve existing reply and review-status links; nested FDF export uses lazy array owners and annotation positions, with retained entries budgeted before cloning under copied-graph limits and source `/Page` and `/Pages` dictionaries replaced by null at every copy depth); flatten; Fill & Sign with adjustable text/date size; agent tools.
Done since: polygons, connected lines, clouds, callouts, inserted text, colour and checkmark filters; image signature pointer previews, selection handles and live image movement/resizing with proportional corners and independent edge handles. A refused restyle or resize of a text box another program drew leaves that comment unchanged (#1327). Make Current Properties Default on an arrow updates arrows, not plain lines (#1302). Flattening keeps a visible comment it cannot draw (#1317). Missing: custom stamps, replace-text proposals, summary layouts with the page | -| M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references. CSV export and spreadsheet merging quote tabs, and import distinguishes quoted tabs from TSV delimiters without changing field identities. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Refreshed against upstream `99941daa` (2026-10-15).
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Done since: Push-button normal labels are editable per widget through Options and form_set_props, independently of field names; rendering, undo/redo and incremental/full save/reopen are covered. (#2892). Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | +| M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references. CSV export and spreadsheet merging quote tabs, and import distinguishes quoted tabs from TSV delimiters without changing field identities. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Refreshed against upstream `99941daa` (2026-10-15).
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Done since: Push-button normal labels are editable per widget through Options and form_set_props, independently of field names; rendering, undo/redo and incremental/full save/reopen are covered. (#2892). Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values.
Done since: Form detection preserves comments across content streams and does not add fields over rectangles hidden by those comments. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Refreshed against upstream `99941daa` (2026-10-15).
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references. CSV export and spreadsheet merging quote tabs, and import distinguishes quoted tabs from TSV delimiters without changing field identities.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | @@ -200,6 +201,7 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st | Date | Change | Summary | |---|---|---| +| 2026-10-20 | minor | M6, #3502: form detection preserves comments across content streams; estimates unchanged. | | 2026-10-18 | minor | M5, PR #1971: nested source `/Page` and `/Pages` dictionaries export as null without traversing page content or the page tree. Synthetic saved/reopened three-page regression retains unrelated metadata and aliases. FDF import semantics, copied-graph limits and estimates unchanged. | | 2026-10-15 | minor | M5: budget retained FDF entries before cloning; discovery keeps lazy array owners and annotation positions, with alias/cache-eviction stress fixtures. Limits cover the copied graph; source parsing/page discovery retain existing COS/parser limits, with no total source-memory bound. FDF import semantics and estimates unchanged. | | 2026-10-15 | minor | M7: Edit text caret height follows its row, with a kittest regression; estimates unchanged |