diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index e4755ab385..fe8eb292c7 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -182,17 +182,51 @@ fn union(a: Option<[f64; 4]>, b: [f64; 4]) -> [f64; 4] { } } -/// Where the marked content `ids` is drawn on a page (images, forms and paths). -fn content_bbox(doc: &Document, page: &Dict, ids: &HashSet) -> Option<[f64; 4]> { +/// An open marked-content sequence while bounds are gathered: its id, when it is one of those asked +/// for, and the bounds of what has been drawn in it so far. +struct Mark { + id: Option, + drawn: Option<[f64; 4]>, +} + +/// Adds `b` to what the innermost open sequence has drawn. +fn draw(marks: &mut [Mark], b: [f64; 4]) { + if let Some(m) = marks.last_mut() { + m.drawn = Some(union(m.drawn, b)); + } +} + +/// Closes the innermost open sequence: what it drew counts for its id and for the sequence around it. +fn close(marks: &mut Vec, matching: &mut usize, out: &mut HashMap) { + let Some(m) = marks.pop() else { return }; + if m.id.is_some() { + *matching = matching.saturating_sub(1); + } + let Some(b) = m.drawn else { return }; + if let Some(id) = m.id { + let e = out.entry(id).or_insert(b); + *e = union(Some(*e), b); + } + draw(marks, b); +} + +/// Where each of the marked content `ids` is drawn on a page (images, forms and paths), in one pass +/// over the page. What a sequence draws counts for every sequence of `ids` around it too. +pub(crate) fn content_bboxes(doc: &Document, page: &Dict, ids: &HashSet) -> HashMap { let res = page.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let xobjects = res.get(b"XObject").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); let props = page_properties(doc, page); + #[cfg(test)] + crate::BOUNDS_PASSES.with(|c| c.set(c.get() + 1)); let mut ctm = vec![Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])]; - let mut marks: Vec = Vec::new(); - let mut bbox = None; + // The open marked-content sequences, and how many of them are one of `ids`. + let mut marks: Vec = Vec::new(); + let mut matching = 0usize; + let mut out = HashMap::new(); let mut path: Vec<(f64, f64)> = Vec::new(); for op in pdfcraft_content::parse(&page_content(doc, page)).ops { - let inside = marks.iter().any(|m| *m); + crate::count_mark_reads(1); + let inside = matching > 0; let top = ctm.last().copied().unwrap_or(Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])); match op.op.as_slice() { b"q" => ctm.push(top), @@ -206,11 +240,13 @@ fn content_bbox(doc: &Document, page: &Dict, ids: &HashSet) -> Option<[f64; *t = m.then(&top); } } - b"BMC" => marks.push(false), - b"BDC" => marks.push(bdc_mcid(doc, &props, &op).is_some_and(|n| ids.contains(&n))), - b"EMC" => { - marks.pop(); + b"BMC" => marks.push(Mark { id: None, drawn: None }), + b"BDC" => { + let id = bdc_mcid(doc, &props, &op).filter(|n| ids.contains(n)); + matching += usize::from(id.is_some()); + marks.push(Mark { id, drawn: None }); } + b"EMC" => close(&mut marks, &mut matching, &mut out), b"m" | b"l" => { if let Some([x, y]) = op.nums::<2>() { path.push(top.apply(x, y)); @@ -231,7 +267,7 @@ fn content_bbox(doc: &Document, page: &Dict, ids: &HashSet) -> Option<[f64; b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s" | b"n" => { if inside && !matches!(op.op.as_slice(), b"n") { for (x, y) in &path { - bbox = Some(union(bbox, [*x, *y, *x, *y])); + draw(&mut marks, [*x, *y, *x, *y]); } } path.clear(); @@ -251,13 +287,16 @@ fn content_bbox(doc: &Document, page: &Dict, ids: &HashSet) -> Option<[f64; } _ => top.bbox([0.0, 0.0, 1.0, 1.0]), }; - bbox = Some(union(bbox, unit)); + draw(&mut marks, unit); } - b"BI" if inside => bbox = Some(union(bbox, top.bbox([0.0, 0.0, 1.0, 1.0]))), + b"BI" if inside => draw(&mut marks, top.bbox([0.0, 0.0, 1.0, 1.0])), _ => {} } } - bbox + while !marks.is_empty() { + close(&mut marks, &mut matching, &mut out); + } + out } /// The figures, in document order. @@ -266,14 +305,28 @@ pub fn figures(doc: &Document) -> Vec
{ let role_map = root.get(b"RoleMap").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default(); let pages = pdfcraft_model::pages(doc); let index: HashMap = pages.iter().enumerate().map(|(i, p)| (p.obj, i)).collect(); - elements(doc) + let figures: Vec<(Element, Option)> = elements(doc) .into_iter() .filter(|e| e.dict.name(b"S").is_some_and(|s| standard(doc, &role_map, s) == b"Figure")) .map(|e| { let page = e.page.and_then(|p| index.get(&p).copied()); + (e, page) + }) + .collect(); + let mut wanted: HashMap> = HashMap::new(); + for (e, page) in &figures { + if let Some(p) = page { + wanted.entry(*p).or_default().extend(mcids(&e.dict)); + } + } + let bounds: HashMap> = + wanted.into_iter().filter_map(|(p, ids)| Some((p, content_bboxes(doc, &pages.get(p)?.dict, &ids)))).collect(); + figures + .into_iter() + .map(|(e, page)| { let alt = e.dict.get(b"Alt").map(|o| doc.resolve(o)).and_then(|o| o.as_string().map(|s| s.to_text())); - let ids: HashSet = mcids(&e.dict).into_iter().collect(); - let bbox = page.and_then(|p| content_bbox(doc, &pages[p].dict, &ids)); + let on_page = page.and_then(|p| bounds.get(&p)); + let bbox = on_page.and_then(|b| mcids(&e.dict).iter().filter_map(|id| b.get(id)).fold(None, |acc, b| Some(union(acc, *b)))); Figure { obj: e.object, page, alt, bbox } }) .collect() diff --git a/crates/a11y/src/headings.rs b/crates/a11y/src/headings.rs index 537ed908d7..41c749ed3c 100644 --- a/crates/a11y/src/headings.rs +++ b/crates/a11y/src/headings.rs @@ -182,6 +182,7 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { let res = sub(doc, page, b"Resources"); let (fonts, props) = (sub(doc, &res, b"Font"), sub(doc, &res, b"Properties")); let mut metrics: HashMap, Metrics> = HashMap::new(); + // The id each open marked-content sequence is in: its own, else the enclosing one's. let mut stack: Vec> = Vec::new(); let mut out: HashMap = HashMap::new(); // Where each id's last glyph ended, in user space. @@ -191,16 +192,20 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { let mut unsaved = 0usize; let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); for op in pdfcraft_content::parse(&page_content(doc, page)).ops { - let id = stack.iter().rev().find_map(|m| *m); + crate::count_mark_reads(1); + let id = stack.last().copied().flatten(); match op.op.as_slice() { - b"BMC" => stack.push(None), - b"BDC" => stack.push(match op.operands.get(1) { - Some(Object::Dict(d)) => d.get(b"MCID").and_then(Object::as_int), - Some(Object::Name(n)) => { - props.get(n).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().and_then(|d| d.get(b"MCID").and_then(Object::as_int))) + b"BMC" => stack.push(id), + b"BDC" => stack.push( + match op.operands.get(1) { + Some(Object::Dict(d)) => d.get(b"MCID").and_then(Object::as_int), + Some(Object::Name(n)) => { + props.get(n).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().and_then(|d| d.get(b"MCID").and_then(Object::as_int))) + } + _ => None, } - _ => None, - }), + .or(id), + ), b"EMC" => { stack.pop(); } @@ -280,17 +285,17 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { }; for (code, len) in m.codes(&s.bytes) { let trm = tm.then(&gs.ctm); + // A glyph's text is only built while its id still takes more. if let Some(id) = id + && out.get(&id).is_none_or(|t| t.len() < MAX_TITLE * 4) && let Some(glyph) = m.text_of(code) { let text = out.entry(id).or_default(); - if text.len() < MAX_TITLE * 4 { - let breaks = ends.get(&id).is_some_and(|end| new_word(&trm, *end, gs.size, gs.scale)); - if breaks && !text.is_empty() && !text.ends_with(' ') && !glyph.starts_with(' ') { - text.push(' '); - } - text.push_str(&glyph); + let breaks = ends.get(&id).is_some_and(|end| new_word(&trm, *end, gs.size, gs.scale)); + if breaks && !text.is_empty() && !text.ends_with(' ') && !glyph.starts_with(' ') { + text.push(' '); } + text.push_str(&glyph); } let advance = m.width(code) * gs.size + gs.char_spacing + if m.is_space(code, len) { gs.word_spacing } else { 0.0 }; tm = Matrix::translate(advance * gs.scale, 0.0).then(&tm); diff --git a/crates/a11y/src/lib.rs b/crates/a11y/src/lib.rs index ff81cc4c97..21d8ea5820 100644 --- a/crates/a11y/src/lib.rs +++ b/crates/a11y/src/lib.rs @@ -21,6 +21,22 @@ mod report; mod structure; mod tags; +#[cfg(test)] +thread_local! { + /// Marked-content stack entries read to decide which marks enclose an operator, for the tests + /// that keep that constant per operator. + static MARK_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Page content passes made to find figure bounds. + static BOUNDS_PASSES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts `n` marked-content stack reads (tests only). +#[inline] +fn count_mark_reads(_n: usize) { + #[cfg(test)] + MARK_READS.with(|c| c.set(c.get().saturating_add(_n))); +} + pub use alt::{AltError, ArtifactError, Figure, figures, mark_decorative, mark_tag_artifact, set_alt}; pub use autotag::{AutoTagError, AutoTagResult, autotag_text}; pub use autotag_fields::tag_form_fields; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 15ca643e83..ad57477364 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1023,6 +1023,67 @@ fn glyph_by_tm(text: &str, size: f64, y: f64) -> String { ops + " ET" } +/// Marked-content stack reads while `f` runs. +fn mark_reads(f: impl FnOnce()) -> usize { + MARK_READS.with(|c| c.set(0)); + f(); + MARK_READS.with(|c| c.get()) +} + +/// Page content nesting `depth` unmarked `BMC` blocks around a marked one, with `ops` operators inside. +fn deeply_nested_page(doc: &mut Document, depth: usize, ops: usize) -> pdfcraft_cos::Dict { + use pdfcraft_cos::{Dict, Stream}; + let mut data = "/Figure << /MCID 0 >> BDC ".to_string(); + data.push_str(&"/X BMC ".repeat(depth)); + data.push_str(&"0 0 1 1 re f ".repeat(ops)); + data.push_str(&"EMC ".repeat(depth + 1)); + let content = doc.add(Stream::flate(Dict::new(), data.as_bytes())); + [(b"Contents".to_vec(), Object::Ref(content))].into_iter().collect() +} + +#[test] +fn marked_text_reads_its_enclosing_id_in_constant_steps() { + let mut doc = Document::new_empty(); + let page = deeply_nested_page(&mut doc, 5_000, 5_000); + let reads = mark_reads(|| { + crate::headings::mcid_texts(&doc, &page); + }); + // About 15 000 operators; a scan of the stack per operator reads millions of entries. + assert!(reads <= 4 * 15_000, "{reads} stack reads"); +} + +#[test] +fn figure_bounds_read_the_marked_content_state_in_constant_steps() { + let mut doc = Document::new_empty(); + let page = deeply_nested_page(&mut doc, 5_000, 5_000); + // No mark encloses figure 7, so a scan of the stack per operator reads all of it. + let reads = mark_reads(|| { + crate::alt::content_bboxes(&doc, &page, &std::collections::HashSet::from([7])); + }); + assert!(reads <= 4 * 15_000, "{reads} stack reads"); +} + +#[test] +fn figure_bounds_take_one_pass_over_each_page() { + let n = 50; + let content: Vec = (0..n).map(|i| format!("/Figure << /MCID {i} >> BDC {i} 0 1 1 re f EMC")).collect(); + let kids: Vec = (0..n).map(|i| format!("{} 0 R", 6 + i)).collect(); + let mut objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 5 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>".into(), + stream(&content.join(" ")), + format!("<< /Type /StructTreeRoot /K [{}] >>", kids.join(" ")), + ]; + objs.extend((0..n).map(|i| format!("<< /S /Figure /Pg 3 0 R /K {i} >>"))); + let doc = pdf(&objs, ""); + BOUNDS_PASSES.with(|c| c.set(0)); + let f = figures(&doc); + assert_eq!(BOUNDS_PASSES.with(|c| c.get()), 1, "page passes"); + assert_eq!(f.len(), n); + assert!(f.iter().enumerate().all(|(i, f)| f.bbox == Some([i as f64, 0.0, i as f64 + 1.0, 1.0]))); +} + /// One tagged page whose H1 headings show `contents` (one marked-content sequence each). fn heading_page(contents: &[String]) -> Document { let marked: Vec = contents.iter().enumerate().map(|(i, c)| format!("/H1 << /MCID {i} >> BDC {c} EMC")).collect(); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index f3030ad58d..8c3999712f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -8,6 +8,8 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +use std::collections::HashMap; + mod pptx; mod zip; @@ -85,14 +87,33 @@ enum Item<'a> { PageBreak, } +#[cfg(test)] +thread_local! { + /// Entries read to find a font size's or a colour's slot, for the tests that keep it constant. + static LOOKUP_STEPS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts `n` entries read by a lookup (tests only). +#[inline] +fn count_lookup_steps(_n: usize) { + #[cfg(test)] + LOOKUP_STEPS.with(|c| c.set(c.get().saturating_add(_n))); +} + /// The body text size: the size most of the document's characters are set in. fn body_size(pages: &[Page]) -> f64 { + // Sizes in first-seen order (which breaks ties), and where each one is in it. let mut by: Vec<(i64, usize)> = Vec::new(); + let mut slot: HashMap = HashMap::new(); for b in pages.iter().flat_map(|p| &p.blocks) { let k = (b.size * 2.0).round() as i64; - match by.iter_mut().find(|e| e.0 == k) { - Some(e) => e.1 += b.text.len(), - None => by.push((k, b.text.len())), + count_lookup_steps(1); + let i = *slot.entry(k).or_insert_with(|| { + by.push((k, 0)); + by.len() - 1 + }); + if let Some(e) = by.get_mut(i) { + e.1 = e.1.saturating_add(b.text.len()); } } by.into_iter().max_by_key(|e| e.1).map_or(11.0, |e| e.0 as f64 / 2.0) @@ -831,9 +852,12 @@ pub fn rtf(pages: &[Page]) -> String { let items = items(pages); // The colour table: entry 0 is the default (auto) colour, then each non-black colour once. let mut colors: Vec<[u8; 3]> = Vec::new(); + let mut slot: HashMap<[u8; 3], usize> = HashMap::new(); let mut note = |c: [f64; 3]| { let c = rgb8(c); - if c != [0, 0, 0] && !colors.contains(&c) { + count_lookup_steps(1); + if c != [0, 0, 0] && !slot.contains_key(&c) { + slot.insert(c, colors.len()); colors.push(c); } }; @@ -847,7 +871,8 @@ pub fn rtf(pages: &[Page]) -> String { // `\cfN` for a colour (nothing for black, which keeps the default). let cf = |c: [f64; 3]| { let c = rgb8(c); - colors.iter().position(|e| *e == c).map_or(String::new(), |i| format!("\\cf{}", i + 1)) + count_lookup_steps(1); + slot.get(&c).map_or(String::new(), |i| format!("\\cf{}", i + 1)) }; let mut s = String::from("{\\rtf1\\ansi\\deff0{\\fonttbl{\\f0 Helvetica;}}\n"); if !colors.is_empty() { @@ -911,6 +936,46 @@ pub fn rtf(pages: &[Page]) -> String { mod tests { use super::*; + /// A page of `n` one-word blocks, each in its own size and colour. + fn varied_page(n: usize) -> Page { + let blocks = (0..n) + .map(|i| Block { + text: "word".into(), + rect: [72.0, i as f64, 100.0, i as f64 + 1.0], + size: 4.0 + i as f64 * 0.5, + bold: false, + italic: false, + color: [(i % 256) as f64 / 255.0, (i / 256 % 256) as f64 / 255.0, 0.5], + rotation: 0.0, + }) + .collect(); + Page { width: 612.0, height: 792.0, blocks, images: Vec::new(), background: None } + } + + fn lookup_steps(f: impl FnOnce()) -> usize { + LOOKUP_STEPS.with(|c| c.set(0)); + f(); + LOOKUP_STEPS.with(|c| c.get()) + } + + #[test] + fn many_distinct_sizes_are_counted_in_constant_steps_each() { + let pages = [varied_page(3000)]; + let steps = lookup_steps(|| { + body_size(&pages); + }); + assert!(steps <= 2 * 3000, "{steps} steps"); + } + + #[test] + fn many_distinct_colours_take_constant_steps_each() { + let pages = [varied_page(3000)]; + let mut out = String::new(); + let steps = lookup_steps(|| out = rtf(&pages)); + assert!(steps <= 4 * 3000, "{steps} steps"); + assert!(out.contains("\\cf3000 ") || out.contains("\\cf3000\\"), "the last colour has the last slot"); + } + fn page() -> Page { let b = |t: &str, y: f64, size: f64, bold: bool| Block { text: t.into(), diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index ec103ed5f0..37a2ea74d8 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -88,17 +88,28 @@ pub fn wrap(text: &str, size: f64, width: f64) -> Vec { } /// [`wrap`], measuring with `measure` (for example [`cjk_width`]). +/// +/// A line measures as the sum of its words, the spaces between them and the characters of a broken +/// word, each measured once, so the work stays linear in `text` however much fits on a line. pub fn wrap_with(text: &str, size: f64, width: f64, measure: impl Fn(&str, f64) -> f64) -> Vec { let mut lines = Vec::new(); + let space = measure(" ", size); + let mut buf = [0u8; 4]; // A CRLF pair is one break, a lone CR or LF is one break each. for para in text.replace("\r\n", "\n").split(['\n', '\r']) { let mut line = String::new(); + let mut line_width = 0.0; // No word is on the line yet; an empty word (a leading space) still starts it. let mut fresh = true; for word in para.split(' ') { - let candidate = if fresh { word.to_string() } else { format!("{line} {word}") }; - if measure(&candidate, size) <= width || fresh && measure(word, size) <= width { - line = candidate; + let word_width = measure(word, size); + let candidate = if fresh { word_width } else { line_width + space + word_width }; + if candidate <= width { + if !fresh { + line.push(' '); + } + line.push_str(word); + line_width = candidate; fresh = false; continue; } @@ -108,11 +119,15 @@ pub fn wrap_with(text: &str, size: f64, width: f64, measure: impl Fn(&str, f64) } else { lines.push(std::mem::take(&mut line)); } + line_width = 0.0; for ch in word.chars() { - if !line.is_empty() && measure(&format!("{line}{ch}"), size) > width { + let ch_width = measure(ch.encode_utf8(&mut buf), size); + if !line.is_empty() && line_width + ch_width > width { lines.push(std::mem::take(&mut line)); + line_width = 0.0; } line.push(ch); + line_width += ch_width; } fresh = line.is_empty(); } @@ -442,4 +457,20 @@ mod tests { let lines = wrap_with("日本語のテキストは文字単位で折り返されます", 10.0, 45.0, cjk_width); assert!(lines.len() > 3 && lines.iter().all(|l| cjk_width(l, 10.0) <= 45.0), "{lines:?}"); } + + /// Wrapping measures each word and character a bounded number of times, however many fit on a line. + #[test] + fn wrapping_measures_each_word_a_few_times() { + let measured = std::cell::Cell::new(0usize); + let measure = |s: &str, size: f64| { + measured.set(measured.get() + s.len()); + helvetica_width(s, size) + }; + let text = format!("{} {}", "word ".repeat(5_000), "x".repeat(5_000)); + let lines = wrap_with(&text, 1.0, 13_000.0, measure); + assert_eq!(lines.len(), 2, "the words fit on one line, the long word on another"); + assert!(measured.get() <= 4 * text.len(), "{} bytes measured", measured.get()); + let narrow = wrap_with(&text, 10.0, 100.0, helvetica_width); + assert!(narrow.iter().all(|l| helvetica_width(l, 10.0) <= 100.0), "{narrow:?}"); + } } diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index a5e7b3a18e..a083a36329 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -33,9 +33,9 @@ pub struct Metrics { widths: Vec, /// Composite fonts: CID → width, and CID ranges with one width. cid_widths: HashMap, - cid_ranges: Vec<(u32, u32, f64)>, + cid_ranges: Spans, /// Code → CID for embedded non-identity CMaps (`cidrange` / `cidchar`). - cid_map: Vec<(u32, u32, u32)>, + cid_map: Spans, default: f64, /// Composite fonts with an explicit `/DW`: `/W` may leave out the glyphs that have that /// width (Chrome and Skia do), so a CID missing from `/W` is not a missing glyph. @@ -598,8 +598,8 @@ impl Metrics { first: 0, widths: Vec::new(), cid_widths: HashMap::new(), - cid_ranges: Vec::new(), - cid_map: Vec::new(), + cid_ranges: Spans::default(), + cid_map: Spans::default(), default: 500.0, explicit_default: false, std14: Some(crate::Std14::Helvetica), @@ -685,7 +685,7 @@ impl Metrics { if !spaces.is_empty() { m.codes = Codes::Ranges(spaces); } - m.cid_map = cids; + m.cid_map = Spans::new(cids); } let desc = font .get(b"DescendantFonts") @@ -698,6 +698,7 @@ impl Metrics { m.explicit_default = true; } if let Some(w) = desc.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + let mut ranges = Vec::new(); let mut i = 0; while i < w.len() { let Some(c0) = doc.resolve(&w[i]).as_f64().map(|v| v as u32) else { break }; @@ -713,14 +714,17 @@ impl Metrics { i += 2; } Some(c1) => { - if let (Some(c1), Some(v)) = (c1.as_f64(), w.get(i + 2).and_then(|x| doc.resolve(x).as_f64())) { - m.cid_ranges.push((c0, c1 as u32, v)); + if let (Some(c1), Some(v)) = (c1.as_f64(), w.get(i + 2).and_then(|x| doc.resolve(x).as_f64())) + && ranges.len() < MAX_CMAP_ENTRIES + { + ranges.push((c0, c1 as u32, v)); } i += 3; } None => break, } } + m.cid_ranges = Spans::new(ranges); } descriptor = dict(doc, desc.get(b"FontDescriptor")); } else { @@ -791,14 +795,14 @@ impl Metrics { if self.cid_map.is_empty() { return code; } - self.cid_map.iter().find(|(lo, hi, _)| code >= *lo && code <= *hi).map_or(0, |(lo, _, c)| c.saturating_add(code - lo)) + self.cid_map.find(code).map_or(0, |(lo, _, c)| c.saturating_add(code.saturating_sub(*lo))) } /// The advance of `code` in text space per unit of font size. pub fn width(&self, code: u32) -> f64 { if self.composite { let cid = self.cid(code); - let w = self.cid_widths.get(&cid).copied().or_else(|| self.cid_ranges.iter().find(|(a, b, _)| cid >= *a && cid <= *b).map(|r| r.2)); + let w = self.cid_widths.get(&cid).copied().or_else(|| self.cid_ranges.find(cid).map(|r| r.2)); return w.unwrap_or(self.default) * self.scale; } if let Some(w) = code.checked_sub(self.first).and_then(|i| self.widths.get(i as usize)) { @@ -858,7 +862,7 @@ impl Metrics { // A CID /W leaves out takes /DW, and Skia (Chrome) leaves out every CID of that width: for // those, only the subset's program says whether the glyph is there. return self.cid_widths.contains_key(&cid) - || self.cid_ranges.iter().any(|(a, b, _)| cid >= *a && cid <= *b) + || self.cid_ranges.find(cid).is_some() || !self.subset || self.outlined_cids.get(cid as usize / 64).is_some_and(|w| (*w >> (cid % 64)) & 1 == 1) // No readable TrueType program to ask (e.g. CFF or missing): an explicit /DW @@ -905,9 +909,13 @@ impl Metrics { } } -/// The UTF-16 code units of CMap hex bytes. +/// Most UTF-16 units kept from one ToUnicode destination: the CMap format allows destinations of up +/// to 512 bytes, and every glyph a sequential range covers builds its text from the whole of one. +const MAX_DESTINATION_UNITS: usize = 256; + +/// The UTF-16 code units of CMap hex bytes, at most [`MAX_DESTINATION_UNITS`] of them. fn code_units(b: &[u8]) -> Vec { - b.chunks(2).map(|c| u16::from_be_bytes([c[0], *c.get(1).unwrap_or(&0)])).collect() + b.chunks(2).take(MAX_DESTINATION_UNITS).map(|c| u16::from_be_bytes([c[0], *c.get(1).unwrap_or(&0)])).collect() } /// UTF-16BE (with surrogates) from CMap hex bytes. @@ -954,16 +962,69 @@ fn add_range(out: &mut Unicode, budget: &mut usize, seq: &mut u32, lo: u32, hi: true } +#[cfg(test)] +thread_local! { + /// Code span records read by CID and width lookups, for the tests that keep a lookup logarithmic. + static SPAN_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts one span record read (tests only). +#[inline] +fn count_span_reads() { + #[cfg(test)] + SPAN_READS.with(|c| c.set(c.get().saturating_add(1))); +} + +/// Code spans `(lo, hi, value)` in declaration order, where the first-declared span covering a code +/// wins, indexed so that a lookup is one binary search. +#[derive(Clone, Debug, Default, PartialEq)] +struct Spans { + spans: Vec<(u32, u32, T)>, + /// The span covering each stretch of codes; see [`stretches`]. + cover: Vec<(u64, Option)>, +} + +impl Spans { + fn new(spans: Vec<(u32, u32, T)>) -> Self { + let bounds: Vec<(u32, u32)> = spans.iter().map(|s| (s.0, s.1)).collect(); + Spans { cover: stretches(&bounds, false), spans } + } + + fn is_empty(&self) -> bool { + self.spans.is_empty() + } + + /// The first-declared span covering `code`. + fn find(&self, code: u32) -> Option<&(u32, u32, T)> { + let step = self.cover.partition_point(|&(start, _)| { + count_span_reads(); + start <= u64::from(code) + }); + let &(_, record) = self.cover.get(step.checked_sub(1)?)?; + self.spans.get(record?) + } +} + /// For each stretch of codes, from its first code on, the latest-declared range covering it (`None` /// where no range does), sorted by code; a lookup is one binary search over it. fn cover_index(ranges: &[BfRange]) -> Vec<(u64, Option)> { + let spans: Vec<(u32, u32)> = ranges.iter().map(|r| (r.lo, r.hi)).collect(); + stretches(&spans, true) +} + +/// For each stretch of codes, from its first code on, the span `(lo, hi)` covering it (`None` where +/// none does), sorted by code: the latest-declared one when `later_wins`, else the first-declared. +/// At most `u32::MAX` spans are indexed. +fn stretches(spans: &[(u32, u32)], later_wins: bool) -> Vec<(u64, Option)> { + // Heap keys, largest for the winning record; the mapping is its own inverse. + let key = |record: u32| if later_wins { record } else { !record }; // Openings (first code, record) and closings (one past the last code), each in code order. The cover // can only change at one of their codes, so the sweep visits those codes in order. - let mut opens: Vec<(u32, u32)> = ranges.iter().enumerate().map(|(i, r)| (r.lo, i as u32)).collect(); + let mut opens: Vec<(u32, u32)> = spans.iter().zip(0..u32::MAX).map(|(&(lo, _), i)| (lo, i)).collect(); opens.sort_unstable(); - let mut closes: Vec = ranges.iter().map(|r| u64::from(r.hi) + 1).collect(); + let mut closes: Vec = spans.iter().take(opens.len()).map(|&(_, hi)| u64::from(hi) + 1).collect(); closes.sort_unstable(); - // Records that have opened and not yet closed; the top is the latest declared one. + // Keys of the records that have opened and not yet closed; the top is the winning one. let mut open: BinaryHeap = BinaryHeap::new(); let mut cover: Vec<(u64, Option)> = Vec::new(); let (mut next_open, mut next_close) = (0, 0); @@ -977,16 +1038,16 @@ fn cover_index(ranges: &[BfRange]) -> Vec<(u64, Option)> { while let Some(&(lo, record)) = opens.get(next_open) && u64::from(lo) <= p { - open.push(record); + open.push(key(record)); next_open += 1; } while closes.get(next_close).is_some_and(|&end| end <= p) { next_close += 1; } - while open.peek().is_some_and(|&top| ranges.get(top as usize).is_none_or(|r| u64::from(r.hi) < p)) { + while open.peek().is_some_and(|&top| spans.get(key(top) as usize).is_none_or(|&(_, hi)| u64::from(hi) < p)) { open.pop(); } - let winner = open.peek().map(|&top| top as usize); + let winner = open.peek().map(|&top| key(top) as usize); if cover.last().map(|&(_, w)| w) != Some(winner) { cover.push((p, winner)); } @@ -1262,6 +1323,88 @@ mod tests { use super::*; + /// Span records read while `f` runs. + fn span_reads(f: impl FnOnce()) -> usize { + SPAN_READS.with(|c| c.set(0)); + f(); + SPAN_READS.with(|c| c.get()) + } + + /// A composite font with `encoding` as its CMap, `w` as its widths and `to_unicode`. + fn composite(doc: &mut Document, encoding: &str, w: Vec, to_unicode: &str) -> Dict { + let mut desc = Dict::new(); + desc.set(b"Subtype".to_vec(), Object::name("CIDFontType2")); + desc.set(b"W".to_vec(), Object::Array(w)); + let enc = Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), encoding.as_bytes())))); + let tu = Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), to_unicode.as_bytes())))); + font( + doc, + vec![ + ("Subtype", Object::name("Type0")), + ("Encoding", enc), + ("DescendantFonts", Object::Array(vec![Object::Dict(desc)])), + ("ToUnicode", tu), + ], + ) + } + + /// Many `/W` ranges that leave a code out cost a glyph of it a binary search, not a scan of them all. + #[test] + fn widths_of_a_code_no_range_covers_take_few_steps() { + let mut doc = Document::new_empty(); + let w: Vec = (1..=10_000i64).flat_map(|k| [Object::Int(2 * k), Object::Int(2 * k), Object::Int(500)]).collect(); + let f = composite(&mut doc, "", w, ""); + let m = Metrics::from_dict(&doc, &f); + assert_eq!(m.width(4), 0.5); + let reads = span_reads(|| { + for _ in 0..1000 { + assert_eq!(m.width(3), 1.0); + } + }); + assert!(reads <= 1000 * 64, "{reads} span reads"); + } + + /// Where spans overlap, the first-declared one wins, as a scan in declaration order finds it. + #[test] + fn overlapping_spans_resolve_to_the_first_declared() { + let spans = Spans::new(vec![(0, 10, 'a'), (5, 20, 'b'), (3, 3, 'c'), (30, 25, 'd'), (u32::MAX, u32::MAX, 'e')]); + let found = |code| spans.find(code).map(|s| s.2); + let want = [(0, Some('a')), (3, Some('a')), (10, Some('a')), (11, Some('b')), (20, Some('b')), (21, None), (27, None), (u32::MAX, Some('e'))]; + for (code, value) in want { + assert_eq!(found(code), value, "code {code}"); + } + } + + /// Checking the glyph of every explicit ToUnicode entry looks its code up once in the CID map. + #[test] + fn many_unicode_entries_look_up_their_cids_in_few_steps() { + let n = 2000; + let cidchars: String = (0..n).map(|k| format!("<{:04X}> {} ", 0x8000 + k, k + 1)).collect(); + let encoding = format!("1 begincodespacerange <0000> endcodespacerange {n} begincidchar {cidchars} endcidchar"); + let bfchars: String = (0..n).map(|k| format!("<{:04X}> <{:04X}> ", 0x1000 + k, 0x4E00 + k)).collect(); + let to_unicode = format!("{n} beginbfchar {bfchars} endbfchar"); + let mut doc = Document::new_empty(); + let f = composite(&mut doc, &encoding, Vec::new(), &to_unicode); + let reads = span_reads(|| { + Metrics::from_dict(&doc, &f); + }); + assert!(reads <= n * 64, "{reads} span reads"); + } + + /// Every glyph of a sequential range builds its text from the destination, so a long one is cut + /// to the CMap limit once, when it is read. + #[test] + fn unicode_destinations_are_cut_to_the_cmap_limit() { + let long = "0041".repeat(4096); + let cmap = format!("2 beginbfrange <0010> <0020> <{long}> <0030> <0030> [<{long}>] endbfrange 1 beginbfchar <0040> <{long}> endbfchar"); + let mut u = Unicode::default(); + parse_to_unicode(cmap.as_bytes(), &mut u); + for code in [0x10, 0x20, 0x30, 0x40] { + let text = u.text(code).unwrap_or_default(); + assert!(!text.is_empty() && text.len() <= MAX_DESTINATION_UNITS * 3, "code {code:#x}: {} bytes", text.len()); + } + } + /// A font whose ranges cover a few low codes keeps tables that size, not a fixed 64k-entry one. #[test] fn low_code_tables_end_with_the_last_covered_code() { @@ -1718,7 +1861,7 @@ mod tests { let map = to_unicode(b"1 beginbfrange <0001> <0003> [<0041> <0043>] endbfrange"); assert_eq!((map.get(&2), map.get(&3).map(String::as_str)), (None, Some("C"))); let mut m = Metrics::fallback(); - (_, m.cid_map) = parse_cmap(b"1 begincidrange <0000> 4294967295 endcidrange"); + m.cid_map = Spans::new(parse_cmap(b"1 begincidrange <0000> 4294967295 endcidrange").1); assert_eq!(m.cid(5), u32::MAX); let mut doc = Document::new_empty(); let mut enc = Dict::new(); diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index caf617f717..44e1655460 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -29,6 +29,19 @@ pub const MIN_FRAGMENTS: usize = 4; /// Most pixels a merged image may have (memory). const MAX_PIXELS: u64 = 80_000_000; +/// Fragment groups a page's search for runs may read per content operator, and at least. A run +/// that is not one picture is searched again from each of its groups, so without a limit images +/// drawn over one another take time quadratic in their number; past it, the rest of the page is +/// left as it is. +const SCAN_PER_OP: usize = 16; +const MIN_SCAN: usize = 1 << 16; + +#[cfg(test)] +thread_local! { + /// Fragment groups read, for the test that keeps a page's search linear in its content. + static GROUP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// One fragment: its image, its box in the coordinates the run is drawn in, and its size. struct Fragment { image: ObjRef, @@ -40,6 +53,8 @@ struct Fragment { /// The ops `q`, one or more `cm`, `Do`, `Q` starting at `i`: the index after them, the matrix /// and the name drawn. fn group_at(ops: &[Op], i: usize) -> Option<(usize, Matrix, Vec)> { + #[cfg(test)] + GROUP_READS.with(|c| c.set(c.get().saturating_add(1))); if !ops.get(i)?.is("q") { return None; } @@ -226,7 +241,13 @@ fn merge_page(doc: &mut Document, page: ObjRef, report: &mut Report, progress: & let ops = parsed.ops; let mut out: Vec = Vec::with_capacity(ops.len()); let (mut i, mut merged, mut k) = (0, 0, 0); + let budget = ops.len().saturating_mul(SCAN_PER_OP).max(MIN_SCAN); + let mut scanned = 0usize; while i < ops.len() { + if scanned > budget { + out.extend(ops.get(i..).unwrap_or_default().iter().cloned()); + break; + } // The longest run of fragment groups from here with one colour space. let mut run = Vec::new(); let mut kind: Option<(Object, usize)> = None; @@ -245,10 +266,12 @@ fn merge_page(doc: &mut Document, page: ObjRef, report: &mut Report, progress: & run.push(Fragment { image, rect: [e, f, e + a, f + d], width, height }); j = next; } + scanned = scanned.saturating_add(run.len()).saturating_add(1); // Not one picture as a whole: the rows drawn one under the other from the start may be. if run.len() >= MIN_FRAGMENTS && !stacked(&mut run) { run = drawn_in_sequence(&ops, i, &xobjects, doc); j = run_end(&ops, i, run.len()); + scanned = scanned.saturating_add(run.len().saturating_mul(2)).saturating_add(2); } let Some((cs, n)) = kind.filter(|_| run.len() >= MIN_FRAGMENTS && stacked(&mut run)) else { // Not a run: keep this op and look again from the next one. @@ -304,6 +327,8 @@ pub(crate) fn run(doc: &mut Document, pages: &[ObjRef], report: &mut Report, pro #[cfg(test)] mod tests { + use pdfcraft_cos::{Dict, Document, Object, Stream}; + use super::*; #[test] @@ -318,4 +343,34 @@ mod tests { ]; assert!(merge(&Document::new_empty(), &run, 3).is_none()); } + + /// Many images drawn on top of each other are not one picture; looking for runs among them reads + /// a bounded number of groups per operator, not the rest of the page from every group. + #[test] + fn rejected_runs_read_a_bounded_number_of_groups() { + let mut doc = Document::new_empty(); + let mut image = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(1)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + image.set(k.as_bytes().to_vec(), v); + } + image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let image = doc.add(Object::Stream(Stream::from_raw(image, vec![0]))); + let groups = 4000; + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), "q 10 0 0 10 0 0 cm /Im Do Q\n".repeat(groups).as_bytes()))); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Contents".to_vec(), Object::Ref(content)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + let page = doc.add(Object::Dict(page)); + GROUP_READS.with(|c| c.set(0)); + let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true); + assert_eq!(merged.ok(), Some(0)); + let reads = GROUP_READS.with(|c| c.get()); + assert!(reads <= 40 * 4 * groups, "{reads} group reads"); + } } diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 6182b08922..1d8a59e253 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -227,6 +227,19 @@ fn is_dct(s: &Stream) -> Option { } } +#[cfg(test)] +thread_local! { + /// Pixels decoded from JPEG data, for the test that keeps a mismatched JPEG from decoding. + static DECODED_PIXELS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts `n` decoded pixels (tests only). +#[inline] +fn count_decoded(_n: u64) { + #[cfg(test)] + DECODED_PIXELS.with(|c| c.set(c.get().saturating_add(_n))); +} + /// 8-bit samples of an image (`n` components), decoded. pub(crate) fn pixels(s: &Stream, n: usize) -> Option> { let (w, h) = (u32::try_from(s.dict.int(b"Width")?).ok()?, u32::try_from(s.dict.int(b"Height")?).ok()?); @@ -235,7 +248,15 @@ pub(crate) fn pixels(s: &Stream, n: usize) -> Option> { } let bytes = usize::try_from(w).ok()?.checked_mul(usize::try_from(h).ok()?)?.checked_mul(n)?; if is_dct(s)? { - let img = image::load_from_memory_with_format(&s.raw, image::ImageFormat::Jpeg).ok()?; + use image::ImageDecoder as _; + // The JPEG's own size is what gets decoded: one other than the dictionary's is refused from + // its header, before any pixels are. + let decoder = image::codecs::jpeg::JpegDecoder::new(std::io::Cursor::new(&s.raw[..])).ok()?; + if decoder.dimensions() != (w, h) { + return None; + } + let img = image::DynamicImage::from_decoder(decoder).ok()?; + count_decoded(u64::from(img.width()) * u64::from(img.height())); // Adobe CMYK/YCCK JPEGs aren't handled (the colour space check excludes them). let v = if n == 1 { img.into_luma8().into_raw() } else { img.into_rgb8().into_raw() }; (v.len() == bytes).then_some(v) @@ -432,4 +453,21 @@ mod dimension_tests { let stream = Stream::from_raw(dict, vec![1, 2, 3, 4, 5]); assert_eq!(pixels(&stream, 1), Some(vec![1, 2, 3, 4])); } + + /// A JPEG larger than its dictionary says is refused from its header, before its pixels are decoded. + #[test] + fn a_jpeg_of_another_size_than_its_dictionary_is_not_decoded() { + let data = jpeg(&vec![128; 1000 * 1000], 1, 1000, 1000, 50).unwrap_or_default(); + let stream = |w: i64, h: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); + Stream::from_raw(dict, data.clone()) + }; + DECODED_PIXELS.with(|c| c.set(0)); + assert!(pixels(&stream(10, 10), 1).is_none()); + assert_eq!(DECODED_PIXELS.with(|c| c.get()), 0, "pixels decoded"); + assert_eq!(pixels(&stream(1000, 1000), 1).map(|p| p.len()), Some(1000 * 1000)); + } }