mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-10-26 19:19:10 -06:00
* M14: bound large-PDF memory use (#307) Inspect large documents lazily, share parser/resource metadata, bound full-save and raster retention, and cache signature candidate discovery safely. Preserve upstream rendering deduplication, pixel alignment, signature previews, and organizer insertion behavior, with regression coverage for their interaction. Validated with 1,336 workspace tests, 193 parser tests, strict workspace Clippy, 30 WASM crate checks, 1x/2x GUI protocols, and same-PDF workflow comparisons. * cos: drop the writer cache-policy probe and its test-only hooks in production paths * cos: the temporary reader keeps parsed objects until a count or owned-bytes budget Since #552 a stream read from the file is a view into the input, so clearing the whole parse cache after every read saved little memory and cost most of the full-save slowdown (every object parsed twice, indirect /Length objects re-parsed per stream). Clear only past OBJECTS entries or STREAM_BYTES of data of the cache's own (decrypted streams, strings). * cos: the writers refuse a document opened with a stream limit (read-only inspection) * cos: a full save fails rather than writing null when an object reads differently the second time * render: parsing is not timed by the watchdog; timing tests with wide margins A slow first parse (a huge or damaged file) made the whole pool fail every page for its lifetime. Parsing, shared or private, is now its own untimed stage, as on main before the shared parser; rendering is timed from when parsing ends. The init-timeout tests become one deterministic test (a 2 s parse against a 10 ms limit, shared and private). Watchdog tests use limits and injected delays ~10x apart instead of 25-50 ms against real renders. Also: saturating retired-worker count, main's stop-flag test runs on private parsers (the first shared timeout is a private retry), set_queue documents that unlisted results are dropped, and retire() documents the permanent per-worker-parser fallback. * render: page numbers in the lazy page tree use checked conversion * render: the lazy inspector is compared with lopdf on named destinations and object streams too * ui: on-screen pages always render; pages turned back to stay cached; stable thumbnail tags - Visible pages and tiles are always admitted and uploaded (the viewport bounds them); the byte and count limits decide only how much prefetch fits. A HiDPI two-up view or a 5K viewport left pages on 'Rendering...' for good. - Pages no longer demanded stay cached, nearest the current page first, while the 128 MiB allowance and the 24-page count have room: paging back no longer re-renders. - Thumbnail density follows the demand count rounded up to a power of two, so rows scrolling in and out of an organize grid don't change every thumbnail's tag and re-render it. - A suspended (background) tab drains its pool's finished results instead of holding them. - Organize drag-and-drop ends on the primary button only. - Tests: the stale-result test uses valid pixels and checks nothing was uploaded; the organizer wheel test runs 40 steps; regressions for the first three fixes. * vendor: mark the hayro-syntax lock and SegmentList patches and list them as (8) and (9); clippy 1.99 as_chunks in a raster test * render: clippy 1.99 — contains over iter().any() in a raster test --------- Co-authored-by: Brandon Thomas <echelon@gmail.com>
218 lines
5.8 KiB
Rust
218 lines
5.8 KiB
Rust
use crate::sync::OnceLock;
|
|
use alloc::boxed::Box;
|
|
use alloc::vec::Vec;
|
|
use core::ops::Sub;
|
|
use smallvec::{Array, SmallVec};
|
|
|
|
pub(crate) trait OptionLog {
|
|
fn error_none(self, f: &str) -> Self;
|
|
}
|
|
|
|
impl<T> OptionLog for Option<T> {
|
|
fn error_none(self, _f: &str) -> Self {
|
|
self.or_else(|| {
|
|
error!("{_f}");
|
|
|
|
None
|
|
})
|
|
}
|
|
}
|
|
|
|
const SCALAR_NEARLY_ZERO: f32 = 1.0 / (1 << 12) as f32;
|
|
|
|
/// A number of useful methods for f32 numbers.
|
|
pub(crate) trait FloatExt: Sized + Sub<f32, Output = f32> + Copy {
|
|
/// Whether the number is approximately 0.
|
|
fn is_nearly_zero(&self) -> bool {
|
|
self.is_nearly_zero_within_tolerance(SCALAR_NEARLY_ZERO)
|
|
}
|
|
|
|
/// Whether the number is approximately 0, with a given tolerance.
|
|
fn is_nearly_zero_within_tolerance(&self, tolerance: f32) -> bool;
|
|
}
|
|
|
|
impl FloatExt for f32 {
|
|
fn is_nearly_zero_within_tolerance(&self, tolerance: f32) -> bool {
|
|
debug_assert!(tolerance >= 0.0, "tolerance must be non-negative");
|
|
|
|
self.abs() <= tolerance
|
|
}
|
|
}
|
|
|
|
pub(crate) trait ByteBuf: Default {
|
|
fn with_capacity(capacity: usize) -> Self;
|
|
fn push(&mut self, byte: u8);
|
|
}
|
|
|
|
impl ByteBuf for Vec<u8> {
|
|
fn with_capacity(capacity: usize) -> Self {
|
|
Self::with_capacity(capacity)
|
|
}
|
|
|
|
fn push(&mut self, byte: u8) {
|
|
Self::push(self, byte);
|
|
}
|
|
}
|
|
|
|
impl<A> ByteBuf for SmallVec<A>
|
|
where
|
|
A: Array<Item = u8>,
|
|
{
|
|
fn with_capacity(capacity: usize) -> Self {
|
|
Self::with_capacity(capacity)
|
|
}
|
|
|
|
fn push(&mut self, byte: u8) {
|
|
Self::push(self, byte);
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "unsafe")]
|
|
pub(crate) fn find_needle(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
memchr::memmem::find(haystack, needle)
|
|
}
|
|
|
|
#[cfg(feature = "unsafe")]
|
|
pub(crate) fn findr_needle(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
memchr::memmem::rfind(haystack, needle)
|
|
}
|
|
|
|
#[cfg(not(feature = "unsafe"))]
|
|
pub(crate) fn find_needle(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
if needle.is_empty() {
|
|
return Some(0);
|
|
}
|
|
|
|
let first = needle[0];
|
|
let mut i = 0;
|
|
|
|
while i + needle.len() <= haystack.len() {
|
|
while haystack.get(i).copied()? != first {
|
|
i += 1;
|
|
if i + needle.len() > haystack.len() {
|
|
return None;
|
|
}
|
|
}
|
|
|
|
if haystack.get(i..)?.starts_with(needle) {
|
|
return Some(i);
|
|
}
|
|
|
|
i += 1;
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
#[cfg(not(feature = "unsafe"))]
|
|
pub(crate) fn findr_needle(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
if needle.is_empty() {
|
|
return Some(haystack.len());
|
|
}
|
|
|
|
if needle.len() > haystack.len() {
|
|
return None;
|
|
}
|
|
|
|
let first = needle[0];
|
|
let mut i = haystack.len() - needle.len();
|
|
|
|
loop {
|
|
while haystack.get(i).copied()? != first {
|
|
i = i.checked_sub(1)?;
|
|
}
|
|
|
|
if haystack.get(i..)?.starts_with(needle) {
|
|
return Some(i);
|
|
}
|
|
|
|
i = i.checked_sub(1)?;
|
|
}
|
|
}
|
|
|
|
/// Allows to store elements at an index with an `&self` reference.
|
|
///
|
|
/// This is similar to a thread-safe arena data structure, but with less moving
|
|
/// parts and no unsafe code. It's based on the segment list presented in
|
|
/// <https://matklad.github.io/2023/04/23/data-oriented-parallel-value-interner.html>
|
|
///
|
|
/// Indices should be used in order. Usage of higher indices implies more memory
|
|
/// usage (even if lower indices are not in use).
|
|
///
|
|
/// The capacity is limited at 2^C - 1. Out-of-range indices return `None` (PdfCraft patch:
|
|
/// upstream panics; #307 reaches this with object-stream slot numbers from the file).
|
|
pub(crate) struct SegmentList<T, const C: usize>([OnceLock<Box<[OnceLock<T>]>>; C]);
|
|
|
|
impl<T, const C: usize> SegmentList<T, C> {
|
|
pub(crate) fn new() -> Self {
|
|
Self(core::array::from_fn(|_| OnceLock::new()))
|
|
}
|
|
|
|
pub(crate) fn get(&self, i: usize) -> Option<&T> {
|
|
let (s, k) = self.locate(i)?;
|
|
let segment = self.0.get(s as usize)?.get()?;
|
|
segment.get(k)?.get()
|
|
}
|
|
|
|
#[track_caller]
|
|
pub(crate) fn get_or_init(&self, i: usize, f: impl FnOnce() -> T) -> Option<&T> {
|
|
let (s, k) = self.locate(i)?;
|
|
let segment = self.0.get(s as usize)?.get_or_init(|| {
|
|
(0..2_usize.pow(s))
|
|
.map(|_| OnceLock::new())
|
|
.collect::<Vec<_>>()
|
|
.into_boxed_slice()
|
|
});
|
|
Some(segment.get(k)?.get_or_init(f))
|
|
}
|
|
|
|
fn locate(&self, i: usize) -> Option<(u32, usize)> {
|
|
let power = i.checked_add(2)?.checked_next_power_of_two()? / 2;
|
|
let s = power.trailing_zeros();
|
|
let k = i - ((1 << s) - 1);
|
|
Some((s, k))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_segment_list() {
|
|
let e = SegmentList::<String, 10>::new();
|
|
|
|
// Ensure that it works.
|
|
for i in 0..500 {
|
|
e.get_or_init(i, || format!("{i}"));
|
|
}
|
|
for i in 0..500 {
|
|
assert_eq!(e.get(i), Some(&format!("{i}")));
|
|
}
|
|
|
|
// Ensure that all slots in the first 8 segments are actually in use,
|
|
// i.e. we didn't overallocate.
|
|
for s in 0..8 {
|
|
assert!(e.0[s].get().unwrap().iter().all(|s| s.get().is_some()));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn segment_list_rejects_out_of_range_indices() {
|
|
let list = SegmentList::<u8, 2>::new();
|
|
assert_eq!(list.get_or_init(2, || 7), Some(&7));
|
|
for index in [3, 20, usize::MAX - 1, usize::MAX] {
|
|
assert_eq!(list.get(index), None);
|
|
assert_eq!(list.get_or_init(index, || 9), None);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_findr_needle() {
|
|
assert_eq!(findr_needle(b"abc foo def foo xyz", b"foo"), Some(12));
|
|
assert_eq!(findr_needle(b"abc", b""), Some(3));
|
|
assert_eq!(findr_needle(b"abc", b"abcd"), None);
|
|
assert_eq!(findr_needle(b"abc", b"z"), None);
|
|
}
|
|
}
|