diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index 26addfd80..3ce312f39 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -539,11 +539,23 @@ impl Automation { }; let mut files = Vec::new(); let mut lowered = Vec::new(); - for p in pages { - let img = ex.image(p, dpi, format).map_err(failed)?; + // A write error stops the export as it is, not as a page error. + let mut write_error = None; + let exported = ex.images(&pages, dpi, format, |p, img| { let path = child(&folder, &format!("{stem}_page_{}.{}", p + 1, format.extension())); - write_atomic(&path, &img)?; + write_atomic(&path, &img).map_err(|e| { + let message = e.to_string(); + write_error = Some(e); + message + })?; files.push(path.to_string_lossy().into_owned()); + Ok(()) + }); + if let Some(e) = write_error { + return Err(e); + } + exported.map_err(failed)?; + for &p in &pages { // A page too large for the renderer at `dpi` is drawn at the most it allows. let used = ex.dpi_used(p, dpi); if used < dpi.clamp(18.0, 1200.0) - 0.5 { diff --git a/crates/engine/examples/export_bench.rs b/crates/engine/examples/export_bench.rs new file mode 100644 index 000000000..41520d2ff --- /dev/null +++ b/crates/engine/examples/export_bench.rs @@ -0,0 +1,61 @@ +//! Times Export ▸ Image over every page of a PDF, for before and after comparisons. +//! +//! Usage: `export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]` (defaults: 150 dpi, PNG, 7 runs). Each +//! run exports every page through [`Exporter::images`] from a fresh exporter, as Export ▸ Image +//! does, and prints its wall time; the summary gives the median and the minimum, the total size +//! and an FNV-1a hash of every file in page order. Equal hashes mean equal output. Build with +//! `--release`; timings are diagnostic, never a CI deadline. + +use std::error::Error; +use std::sync::Arc; +use std::time::Instant; + +use pdfcraft_engine::Session; +use pdfcraft_engine::export::{Exporter, ImageFormat}; + +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let path = args.next().ok_or("usage: export_bench FILE [DPI] [png|jpeg|tiff] [RUNS]")?; + let dpi: f64 = args.next().map_or(Ok(150.0), |s| s.parse())?; + let format = match args.next().as_deref() { + None | Some("png") => ImageFormat::Png, + Some("jpeg" | "jpg") => ImageFormat::Jpeg { quality: 85 }, + Some("tiff" | "tif") => ImageFormat::Tiff, + Some(other) => return Err(format!("unknown format {other:?}").into()), + }; + let runs: usize = args.next().map_or(Ok(7), |s| s.parse())?; + let mut session = Session::new(); + let id = session.open("bench.pdf", None, Arc::new(std::fs::read(&path)?), None)?; + let src = session.get(id).ok_or("the document did not open")?.export_source(); + let pages: Vec = (0..src.pages).collect(); + let mut times = Vec::new(); + let mut summary = None; + for run in 0..runs { + let start = Instant::now(); + let mut exporter = Exporter::from_source(src.clone()); + let (mut hash, mut bytes) = (FNV_OFFSET, 0usize); + exporter.images(&pages, dpi, format, |_, file| { + bytes += file.len(); + for b in file { + hash = (hash ^ u64::from(b)).wrapping_mul(FNV_PRIME); + } + Ok(()) + })?; + let ms = start.elapsed().as_secs_f64() * 1000.0; + println!("run {}: {ms:.1} ms", run + 1); + times.push(ms); + summary = Some((bytes, hash)); + } + times.sort_by(f64::total_cmp); + if let (Some(median), Some(min), Some((bytes, hash))) = (times.get(times.len() / 2), times.first(), summary) { + println!( + "{} pages, {} at {dpi} dpi: median {median:.1} ms, min {min:.1} ms ({runs} runs), {bytes} bytes, hash {hash:016x}", + pages.len(), + format.label() + ); + } + Ok(()) +} diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index 01159df80..1330faf27 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -4,13 +4,52 @@ //! text, from a //! document's working file (so unsaved edits and hidden layers are respected, as on screen). +use std::collections::BTreeMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Condvar, Mutex, PoisonError, mpsc}; + use pdfcraft_render::{PageRenderer, RenderRequest, RequestKind}; use crate::Document; +/// At most this many pages are drawn at once by [`Exporter::images`]. +const MAX_IMAGE_WORKERS: usize = 8; +/// Bytes of rasters [`Exporter::images`] may hold at once: 1 GiB, about 16 Letter pages at 300 dpi. +/// Each worker holds one page (the raster, its RGB copy and the encoded file), so large pages run +/// on fewer workers; a page over the cap still exports, on one worker, as it did before. +const IMAGE_MEMORY: f64 = 1024.0 * 1024.0 * 1024.0; + +/// Page `page` drawn at `dpi` and encoded as `format`, saying it was drawn at `used` dpi. +fn draw_image(renderer: &mut PageRenderer, page: usize, dpi: f64, format: ImageFormat, used: f64) -> Result, String> { + let r = renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() }); + if let Some(e) = r.error { + return Err(format!("page {}: {e}", page + 1)); + } + encode_image(r.width, r.height, &r.rgba, format, used) +} + +/// Tells the workers of [`Exporter::images`] to stop when dropped. +struct StopWorkers<'a> { + stop: &'a AtomicBool, + handed_over: &'a (Mutex, Condvar), +} + +impl Drop for StopWorkers<'_> { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + // Taken under the lock, so a worker between its check and its wait still sees it. + drop(self.handed_over.0.lock().unwrap_or_else(PoisonError::into_inner)); + self.handed_over.1.notify_all(); + } +} + /// Renders and extracts pages of one document state. pub struct Exporter { renderer: PageRenderer, + /// What the renderer draws, for the extra renderers [`Exporter::images`] starts. + bytes: std::sync::Arc>, + config: pdfcraft_render::RenderConfig, pages: usize, /// Each page's displayed size in points, for [`Exporter::dpi_used`]. sizes: Vec<(f32, f32)>, @@ -65,7 +104,13 @@ impl Exporter { /// resolution they allow, as the interactive Export does; [`Exporter::dpi_used`] says at what. /// A source whose `config.reject_oversize` is set refuses such pages instead. pub fn from_source(src: ExportSource) -> Self { - Self { renderer: PageRenderer::new(src.bytes, src.config), pages: src.pages, sizes: src.sizes } + Self { + renderer: PageRenderer::new(src.bytes.clone(), src.config.clone()), + bytes: src.bytes, + config: src.config, + pages: src.pages, + sizes: src.sizes, + } } /// The resolution page `page` is actually exported at when asked for `dpi`: lower than asked @@ -96,11 +141,146 @@ impl Exporter { /// Page `page` as an image file of `format`, which says the resolution it was drawn at. pub fn image(&mut self, page: usize, dpi: f64, format: ImageFormat) -> Result, String> { self.check(page)?; - let r = self.renderer.render(RenderRequest { page, scale: (dpi.clamp(18.0, 1200.0) / 72.0) as f32, ..Default::default() }); - if let Some(e) = r.error { - return Err(format!("page {}: {e}", page + 1)); + let used = self.dpi_used(page, dpi); + draw_image(&mut self.renderer, page, dpi, format, used) + } + + /// Pages `pages` as image files of `format`, handed to `sink` in order with their page + /// number: the same files as [`Exporter::image`] for each page in turn. Encoding a page costs + /// several times what drawing it does, so several pages are drawn and encoded at once on + /// worker threads, each with its own renderer, while `sink` runs on the calling thread. + /// Workers draw at most one page each ahead of `sink`, and fewer run when the pages are + /// large, so memory stays bounded. Stops at the first page that fails or the first error `sink` + /// returns, and returns it; every page before it has been handed to `sink`. + pub fn images( + &mut self, + pages: &[usize], + dpi: f64, + format: ImageFormat, + sink: impl FnMut(usize, Vec) -> Result<(), String>, + ) -> Result<(), String> { + let workers = self.image_workers(pages, dpi); + self.images_on(workers, pages, dpi, format, sink) + } + + /// [`Exporter::images`] with `workers` pages drawn at once (one: in turn, on this thread). + pub(crate) fn images_on( + &mut self, + workers: usize, + pages: &[usize], + dpi: f64, + format: ImageFormat, + mut sink: impl FnMut(usize, Vec) -> Result<(), String>, + ) -> Result<(), String> { + if workers <= 1 { + for &p in pages { + let img = self.image(p, dpi, format)?; + sink(p, img)?; + } + return Ok(()); } - encode_image(r.width, r.height, &r.rgba, format, self.dpi_used(page, dpi)) + let used: Vec = pages.iter().map(|p| self.dpi_used(*p, dpi)).collect(); + let count = self.pages; + let (bytes, config) = (&self.bytes, &self.config); + let next = AtomicUsize::new(0); + let stop = AtomicBool::new(false); + // How many pages `sink` has taken; a worker waits while its page is `workers` past it. + let handed_over = (Mutex::new(0usize), Condvar::new()); + let (tx, rx) = mpsc::channel::<(usize, Result, String>)>(); + let mut first = Some(&mut self.renderer); + std::thread::scope(|s| { + let handles: Vec<_> = (0..workers) + .map(|_| { + let tx = tx.clone(); + let (first, used, next, stop, handed_over) = (first.take(), &used, &next, &stop, &handed_over); + s.spawn(move || { + let mut own = None; + let renderer = match first { + Some(r) => r, + None => own.insert(PageRenderer::new(bytes.clone(), config.clone())), + }; + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let (Some(&page), Some(&used)) = (pages.get(i), used.get(i)) else { break }; + let mut taken = handed_over.0.lock().unwrap_or_else(PoisonError::into_inner); + while i >= taken.saturating_add(workers) && !stop.load(Ordering::Relaxed) { + taken = handed_over.1.wait(taken).unwrap_or_else(PoisonError::into_inner); + } + drop(taken); + if stop.load(Ordering::Relaxed) { + break; + } + let img = if page < count { + catch_unwind(AssertUnwindSafe(|| draw_image(renderer, page, dpi, format, used))) + .unwrap_or_else(|_| Err(format!("page {}: the export crashed", page + 1))) + } else { + Err(format!("page {} does not exist", page + 1)) + }; + if tx.send((i, img)).is_err() { + break; + } + } + }) + }) + .collect(); + drop(tx); + // Stops the workers however this ends, a panicking `sink` included: waiting ones + // would otherwise keep the scope from returning. + let stopper = StopWorkers { stop: &stop, handed_over: &handed_over }; + let mut ready = BTreeMap::new(); + let mut result = Ok(()); + let mut taken = 0; + while let Some(&page) = pages.get(taken) { + let Some(img) = ready.remove(&taken) else { + match rx.recv() { + Ok((i, img)) => { + ready.insert(i, img); + } + Err(_) => { + result = Err(format!("page {}: the export stopped unexpectedly", page + 1)); + break; + } + } + continue; + }; + if let Err(e) = img.and_then(|img| sink(page, img)) { + result = Err(e); + break; + } + taken += 1; + *handed_over.0.lock().unwrap_or_else(PoisonError::into_inner) = taken; + handed_over.1.notify_all(); + } + drop(stopper); + // Joined here, so a worker that panicked outside a page is not re-raised by the scope. + for h in handles { + let _ = h.join(); + } + result + }) + } + + /// How many pages [`Exporter::images`] draws at once: one per core but one, up to + /// [`MAX_IMAGE_WORKERS`], as many as [`IMAGE_MEMORY`] holds of the largest page, and one + /// when there is nothing to share or no threads (wasm). + fn image_workers(&self, pages: &[usize], dpi: f64) -> usize { + if cfg!(target_arch = "wasm32") || pages.len() < 2 { + return 1; + } + // One core is left for the interface, which keeps drawing while an export runs. + let cores = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get).saturating_sub(1).min(MAX_IMAGE_WORKERS); + // The raster (RGBA), the RGB copy the encoders take, and room for the encoded file. + let largest = pages + .iter() + .filter_map(|p| { + let (w, h) = *self.sizes.get(*p)?; + let k = self.dpi_used(*p, dpi) / 72.0; + Some((f64::from(w) * k).ceil() * (f64::from(h) * k).ceil() * 8.0) + }) + .fold(0.0, f64::max); + // A float-to-integer cast saturates (and NaN is 0). + let by_memory = if largest > 0.0 { (IMAGE_MEMORY / largest) as usize } else { cores }; + cores.min(by_memory).min(pages.len()).max(1) } /// The reading-order text of a page. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c622af21b..e7e68a758 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -3410,6 +3410,70 @@ fn pages_export_as_png_jpeg_and_tiff() { assert_eq!((compression, predictor), (5, 1), "LZW, without a predictor"); } +/// Export ▸ Image draws several pages at once: each file is the one exporting that page alone +/// writes, handed over in the order asked for, whatever the number of pages drawn at once. +#[test] +fn pages_exported_at_once_are_the_files_each_page_exports_alone_in_order() { + use crate::export::{Exporter, ImageFormat}; + let (s, id) = session_with(12); + let pages = [3, 0, 11, 5, 5, 7, 1, 2, 10, 9, 8, 6, 4]; + for format in [ImageFormat::Png, ImageFormat::Jpeg { quality: 85 }, ImageFormat::Tiff] { + let alone: Vec<(usize, Vec)> = pages.iter().map(|&p| (p, Exporter::new(s.get(id).unwrap()).image(p, 96.0, format).unwrap())).collect(); + for workers in [1, 2, 4, 8] { + let mut got = Vec::new(); + let mut ex = Exporter::new(s.get(id).unwrap()); + ex.images_on(workers, &pages, 96.0, format, |p, img| { + got.push((p, img)); + Ok(()) + }) + .unwrap(); + assert!(got == alone, "{} with {workers} workers", format.label()); + } + let mut got = Vec::new(); + Exporter::new(s.get(id).unwrap()) + .images(&pages, 96.0, format, |p, img| { + got.push((p, img)); + Ok(()) + }) + .unwrap(); + assert!(got == alone, "{}", format.label()); + } +} + +/// A page that can't be exported, or a file that can't be written, stops the export there with +/// its error: every page before it is handed over, none after it. +#[test] +fn exporting_pages_at_once_stops_at_the_first_error() { + use crate::export::{Exporter, ImageFormat}; + let (s, id) = session_with(6); + for workers in [1, 4] { + let mut ex = Exporter::new(s.get(id).unwrap()); + let mut got = Vec::new(); + let result = ex.images_on(workers, &[0, 1, 99, 2, 3], 72.0, ImageFormat::Png, |p, _| { + got.push(p); + Ok(()) + }); + assert_eq!(result, Err("page 100 does not exist".to_string()), "{workers} workers"); + assert_eq!(got, [0, 1], "{workers} workers"); + + let mut got = Vec::new(); + let result = ex.images_on(workers, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| { + got.push(p); + if p == 2 { Err("disk full".to_string()) } else { Ok(()) } + }); + assert_eq!(result, Err("disk full".to_string()), "{workers} workers"); + assert_eq!(got, [0, 1, 2], "{workers} workers"); + // The exporter is still usable afterwards. + assert!(ex.image(5, 72.0, ImageFormat::Png).is_ok()); + } + // A panicking sink stops the workers rather than leaving them waiting for it. + let mut ex = Exporter::new(s.get(id).unwrap()); + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + ex.images_on(4, &[0, 1, 2, 3, 4, 5], 72.0, ImageFormat::Png, |p, _| if p == 1 { panic!("sink") } else { Ok(()) }) + })); + assert!(panicked.is_err()); +} + /// A TIFF's resolution unit (2 = inch) and its horizontal and vertical resolution. fn tiff_resolution(bytes: &[u8]) -> (u32, f64, f64) { use tiff::{decoder::ifd::Value, tags::Tag}; diff --git a/crates/ui-egui/src/workflows/export_ui.rs b/crates/ui-egui/src/workflows/export_ui.rs index fae075e30..bcda63c57 100644 --- a/crates/ui-egui/src/workflows/export_ui.rs +++ b/crates/ui-egui/src/workflows/export_ui.rs @@ -176,12 +176,18 @@ fn run( // Handled above. ExportKind::AllImages => String::new(), ExportKind::Image => { - for (k, p) in pages.iter().enumerate() { - set(k, None); - let result = ex.image(*p, dpi, format).and_then(|img| sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img)); - if let Err(e) = result { - return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]); - } + set(0, None); + // Pages are drawn several at once and handed over in order: the progress counts the + // pages written so far. + let mut done = 0; + let result = ex.images(&pages, dpi, format, |p, img| { + sink(&format!("{stem}_page_{}.{}", p + 1, format.extension()), img)?; + done += 1; + set(done, None); + Ok(()) + }); + if let Err(e) = result { + return crate::i18n::fmt(tl!("Export stopped: {e}"), &[("e", &e.to_string())]); } let mut msg = if total == 1 { crate::i18n::fmt(tl!("Exported 1 image"), &[])