PDF: put the shown text of fields and note numbers in the text layer (#133)

* PDF: fields and note numbers put their shown text in the text layer

A field, a note reference or an equation is shaped as one cluster that
stands for the object's placeholder character (U+FFFC) in the paragraph
text. The display list took a glyph run's text from the paragraph, so a
footer's "Page 1 of 5" went into the PDF's text layer as "Page  of ":
copying, search and screen readers got the placeholder. 76 of the 365
PDFs written from docxide-pdf's Word reference corpus had it; Word's
have none.

ParaLayout::shown keeps, by cluster index, the text such a cluster
draws (filled by Builder::shape_atomic), and the display list takes that
text for those clusters. The drawing doesn't change.

Now none of the 365 PDFs carries the placeholder; mean line-break
agreement with Word 62.9% -> 63.4% (the scorer reads the text layer).

Test: a footer's PAGE and NUMPAGES fields read "Page 1 of 2" and
"Page 2 of 2" in the exported PDF's text (was "Page  of ").

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Tests: a REF field's result is extractable PDF text (from #112 by @LloydNicholson)

Co-Authored-By: LloydNicholson <LloydNicholson@users.noreply.github.com>

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Oliver Zieschang <4963183+naxvog@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Brandon Thomas <echelon@gmail.com>

crates/layout/src/display.rs

@@ -270,7 +270,12 @@ fn lines(
                     for g in pl.glyphs.get(c.g0 as usize..c.g1 as usize).into_iter().flatten() {
                         glyphs.push((g.gid, cx + g.dx, base - st.shift - g.dy));
                     }
-                    if let Some(p) = para {
+                    // An object's cluster (a field, a note number) stands for the text it shows.
+                    if let Ok(i) = pl.shown.binary_search_by_key(&k, |(i, _)| *i)
+                        && let Some((_, s)) = pl.shown.get(i)
+                    {
+                        text.push_str(s);
+                    } else if let Some(p) = para {
                         text.push_str(p.text.get(c.start..c.end).unwrap_or(""));
                     }
                 }

crates/layout/src/para.rs

@@ -138,6 +138,9 @@ pub struct ParaLayout {
     pub drop_cap: Option<(usize, u8, f32)>,
     /// Clusters after which the line may break with a hyphen (soft hyphens, auto hyphenation), sorted.
     pub hyph_after: Vec<u32>,
+    /// The text drawn by clusters that stand for an object (field results, note numbers,
+    /// equations) rather than for the paragraph's own text, by cluster index, sorted.
+    pub shown: Vec<(usize, String)>,
 }
 
 /// Inputs that change a paragraph's layout beyond its own content.
@@ -177,6 +180,8 @@ struct Builder<'a> {
     style_index: std::collections::HashMap<(String, u32, bool), u16>,
     glyphs: Vec<Glyph>,
     clusters: Vec<Cluster>,
+    /// Text drawn by clusters that stand for an object, by cluster index (see `ParaLayout::shown`).
+    shown: Vec<(usize, String)>,
 }
 
 impl<'a> Builder<'a> {
@@ -326,6 +331,7 @@ impl<'a> Builder<'a> {
         let g0 = first.g0;
         let style = first.style;
         let g1 = added.last().map(|c| c.g1).unwrap_or(g0);
+        self.shown.push((self.clusters.len(), text.to_string()));
         self.clusters.push(Cluster { start, end, adv: x, kind: ClKind::Text, style, g0, g1, break_after: false, obj_h: 0.0, dot: false });
     }
 }
@@ -353,7 +359,7 @@ pub fn layout_para(p: &Paragraph, env: &ParaEnv) -> ParaLayout {
             None => Arc::new(doc.styles.resolve_char(para_style, c)),
         }
     };
-    let mut b = Builder { env, styles: Vec::new(), style_index: Default::default(), glyphs: Vec::new(), clusters: Vec::new() };
+    let mut b = Builder { env, styles: Vec::new(), style_index: Default::default(), glyphs: Vec::new(), clusters: Vec::new(), shown: Vec::new() };
     let mark_rc = resolve(&p.mark);
     let mark_style = b.style(&mark_rc, None, false);
     let mut has_page_fields = false;
@@ -537,6 +543,7 @@ pub fn layout_para(p: &Paragraph, env: &ParaEnv) -> ParaLayout {
         styles: b.styles,
         glyphs: b.glyphs,
         clusters: b.clusters,
+        shown: b.shown,
         lines: Vec::new(),
         label,
         height: 0.0,

crates/pdf/src/tests.rs

@@ -238,6 +238,29 @@ fn shapes_have_paths() {
     assert_eq!(civil(0), (1970, 1, 1, 0, 0, 0));
 }
 
+#[test]
+fn fields_put_their_shown_text_in_the_text_layer() {
+    // "Page 1 of 2" in the footer: the page fields' numbers, not the objects' placeholder
+    // characters, are what copying, search and screen readers get.
+    let mut d = Document::new();
+    let mut f = Paragraph::with_text("Page  of ", CharProps::default());
+    let field = |instr: &str| InlineObject::Field { instr: instr.into(), result: String::new(), locked: false };
+    f.insert_object(5, field("PAGE"), &CharProps::default()).unwrap();
+    let n = f.len();
+    f.insert_object(n, field("NUMPAGES"), &CharProps::default()).unwrap();
+    let id = d.add_part(wordcraft_doc::PartKind::Footer, vec![para_block(f)]);
+    d.last_section.footers.default = Some(id);
+    let mut body = Paragraph::with_text("Body", CharProps::default());
+    body.props.page_break_before = None;
+    let mut second = Paragraph::with_text("More", CharProps::default());
+    second.props.page_break_before = Some(true);
+    d.body = vec![para_block(body), para_block(second)];
+    let text = extract_text(&export(&d, &PdfOptions::default()).unwrap());
+    assert_eq!(text.len(), 2);
+    assert!(text.iter().all(|t| !t.contains('\u{FFFC}')), "{text:?}");
+    assert!(squash(&text[0]).contains("Page 1 of 2") && squash(&text[1]).contains("Page 2 of 2"), "{text:?}");
+}
+
 /// Bundled Source Sans 3 renamed to `family`, with glyph `gid` turned into a composite glyph whose
 /// component list is cut off: fonts still parse and shape, but krilla's subsetter rejects it.
 fn unsubsettable_font(family: &str, gid: u16) -> Vec<u8> {
@@ -337,6 +360,24 @@ fn unsubsettable_font_is_drawn_as_outlines() {
     assert!(paths > 0, "its glyphs are drawn as outlines");
 }
 
+#[test]
+fn field_results_are_extractable_text() {
+    // Issue #97 (test from #112 by @LloydNicholson): a REF field's result, not the U+FFFC
+    // placeholder, is what the text layer gets.
+    let mut d = Document::new();
+    let mut p = Paragraph::with_text("See  for details.", CharProps::default());
+    p.insert_object(
+        4,
+        InlineObject::Field { instr: " REF _RefTarget \\h ".into(), result: "Section 3.01".into(), locked: false },
+        &CharProps::default(),
+    )
+    .unwrap();
+    d.body = vec![para_block(p)];
+    let text = extract_text(&export(&d, &PdfOptions::default()).unwrap()).concat();
+    assert!(!text.contains('\u{FFFC}'), "{text:?}");
+    assert!(squash(&text).contains("See Section 3.01 for details."), "{text:?}");
+}
+
 #[test]
 fn synthetic_bold_text_is_extracted_once() {
     // JetBrains Mono ships without a bold face, so its bold is filled and stroked: the text