From 7a0f846a28108a89f0de7d33c01233b1df4f9bed Mon Sep 17 00:00:00 2001 From: "Sir Clangington III." Date: Tue, 13 Oct 2026 06:12:28 +0200 Subject: [PATCH] M13: Run conversion, OCR and optimization over folder trees (#1605) Add a shared CLI/MCP batch runner with bounded discovery, retained relative paths and per-file processing in isolated engine sessions. Publish complete outputs without overwriting existing names, and return structured errors with an optional stop-on-error policy. Cover recursive jobs, overlapping inputs, concurrent output collisions, root confinement and partial-failure reporting in automation and CLI tests. Co-authored-by: LabDennis <341377186+LabDennis@users.noreply.github.com> --- Cargo.lock | 1 + apps/pdfcraft-cli/src/batch.rs | 61 +++++++ apps/pdfcraft-cli/src/main.rs | 6 + apps/pdfcraft-cli/tests/batch.rs | 46 ++++++ crates/automation/Cargo.toml | 1 + crates/automation/README.md | 6 + crates/automation/src/a11y.rs | 2 +- crates/automation/src/batch.rs | 276 +++++++++++++++++++++++++++++++ crates/automation/src/lib.rs | 2 + crates/automation/src/mcp.rs | 2 +- crates/automation/src/tools.rs | 11 ++ crates/automation/tests/batch.rs | 201 ++++++++++++++++++++++ docs/cli-batch.md | 86 ++++++++++ parity/acrobat-features.toml | 6 +- 14 files changed, 702 insertions(+), 5 deletions(-) create mode 100644 apps/pdfcraft-cli/src/batch.rs create mode 100644 apps/pdfcraft-cli/tests/batch.rs create mode 100644 crates/automation/src/batch.rs create mode 100644 crates/automation/tests/batch.rs create mode 100644 docs/cli-batch.md diff --git a/Cargo.lock b/Cargo.lock index c65431ea0..298258ca6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3632,6 +3632,7 @@ dependencies = [ "png 0.18.1", "serde_json", "sha2", + "tempfile", ] [[package]] diff --git a/apps/pdfcraft-cli/src/batch.rs b/apps/pdfcraft-cli/src/batch.rs new file mode 100644 index 000000000..5d9a70044 --- /dev/null +++ b/apps/pdfcraft-cli/src/batch.rs @@ -0,0 +1,61 @@ +//! Friendly flags for the same folder-job tool exposed by `run` and MCP. + +use serde_json::json; + +use crate::{CliError, OptionSpec, automation, flag, print_output, validate_options}; + +const OPTIONS: &[OptionSpec] = &[ + OptionSpec { name: "--root", takes_value: true }, + OptionSpec { name: "--out-dir", takes_value: true }, + OptionSpec { name: "--steps", takes_value: true }, + OptionSpec { name: "--recursive", takes_value: false }, + OptionSpec { name: "--stop-on-error", takes_value: false }, + OptionSpec { name: "--ppi", takes_value: true }, + OptionSpec { name: "--quality", takes_value: true }, + OptionSpec { name: "--dpi", takes_value: true }, + OptionSpec { name: "--language", takes_value: true }, + OptionSpec { name: "--ocr-all-pages", takes_value: false }, +]; + +pub(super) fn run(args: &[String]) -> Result<(), CliError> { + validate_options("batch", args, OPTIONS)?; + let mut paths = Vec::new(); + let mut seen = std::collections::HashSet::new(); + let mut i = 0; + while let Some(arg) = args.get(i) { + if let Some(spec) = OPTIONS.iter().find(|s| s.name == arg) { + if !seen.insert(spec.name) { + return Err(format!("batch: {} may only be supplied once", spec.name).into()); + } + if spec.takes_value { + if args.get(i + 1).is_none_or(|v| v.is_empty() || v.starts_with("--")) { + return Err(format!("batch: {} needs a value", spec.name).into()); + } + i += 1; + } + } else { + paths.push(arg); + } + i += 1; + } + let steps: Vec<_> = flag(args, "--steps").ok_or("batch: missing --steps (convert, ocr, optimize)")?.split(',').collect(); + let out = flag(args, "--out-dir").ok_or("batch: missing --out-dir")?; + let mut params = + json!({"paths":paths,"steps":steps,"out_dir":out,"recursive":seen.contains("--recursive"),"stop_on_error":seen.contains("--stop-on-error")}); + for (option, key) in [("--ppi", "ppi"), ("--quality", "quality"), ("--dpi", "dpi")] { + if let Some(v) = flag(args, option) { + params[key] = serde_json::from_str(v).map_err(|_| format!("batch: {option} needs a number"))?; + } + } + if let Some(lang) = flag(args, "--language") { + params["language"] = json!(lang); + } + if seen.contains("--ocr-all-pages") { + params["skip_text_pages"] = json!(false); + } + let mut auto = automation(args)?; + let content = auto.call("doc_batch", ¶ms).map_err(|e| e.to_string())?; + let failed = content.iter().any(|c| matches!(c, pdfcraft_automation::Content::Json(v) if v["failed"].as_u64().is_some_and(|n| n > 0))); + print_output(&auto, content, None)?; + if failed { Err("batch: one or more files failed; see the JSON report".into()) } else { Ok(()) } +} diff --git a/apps/pdfcraft-cli/src/main.rs b/apps/pdfcraft-cli/src/main.rs index 818928583..4add709ec 100644 --- a/apps/pdfcraft-cli/src/main.rs +++ b/apps/pdfcraft-cli/src/main.rs @@ -17,6 +17,8 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod batch; + use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode, Stdio}; @@ -57,6 +59,9 @@ pdfcraft-cli extract --pages 1,3,5 --out out.pdf pdfcraft-cli split (--every N | --before 3,7) [--out-dir DIR] pdfcraft-cli check [--timeout 20] [--dpi 36] [--json out.json] pdfcraft-cli audit [--password PW] find faux redactions (fails when found) +pdfcraft-cli batch --steps convert,ocr,optimize --out-dir DIR + [--recursive] [--root DIR] [--stop-on-error] + [--ppi 150] [--quality 60] [--dpi 300] [--language en] [--ocr-all-pages] pdfcraft-cli tools automation tools and their JSON Schemas pdfcraft-cli run [key=value …] [--root DIR] [--out image.png] pdfcraft-cli run --script steps.json [--root DIR] [{\"tool\": \"doc_open\", \"args\": {…}}, …] @@ -90,6 +95,7 @@ fn main() -> ExitCode { Some("audit") => audit(&args[1..]), Some("tools") => tools(), Some("run") => run(&args[1..]), + Some("batch") => batch::run(&args[1..]), Some("ui") => ui(&args[1..]), #[cfg(feature = "mcp")] Some("mcp") => mcp(&args[1..]), diff --git a/apps/pdfcraft-cli/tests/batch.rs b/apps/pdfcraft-cli/tests/batch.rs new file mode 100644 index 000000000..1a751f66a --- /dev/null +++ b/apps/pdfcraft-cli/tests/batch.rs @@ -0,0 +1,46 @@ +use serde_json::Value; +use std::process::Command; + +const BIN: &str = env!("CARGO_BIN_EXE_pdfcraft-cli"); + +#[test] +fn cli_batch_reports_partial_failure_and_continues_into_subfolders() { + let dir = std::env::temp_dir().join(format!("pdfcraft-cli-batch-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("input/sub")).unwrap(); + std::fs::write(dir.join("input/0.pdf"), b"broken").unwrap(); + std::fs::write(dir.join("input/sub/page.txt"), "Batch text").unwrap(); + let output = Command::new(BIN) + .args(["batch", "input", "--root"]) + .arg(&dir) + .args(["--steps", "convert,optimize", "--out-dir", "out", "--recursive"]) + .output() + .unwrap(); + assert!(!output.status.success()); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["completed"], 1, "{report}"); + assert_eq!(report["failed"], 1); + let converted = dir.join("out/sub/page.pdf"); + assert!(converted.exists()); + let text = Command::new(BIN).arg("text").arg(converted).output().unwrap(); + assert!(text.status.success()); + assert!(String::from_utf8_lossy(&text.stdout).contains("Batch text")); + let success = Command::new(BIN).current_dir(&dir).args(["batch", "input/sub", "--steps", "convert", "--out-dir", "other"]).output().unwrap(); + assert!(success.status.success(), "{}", String::from_utf8_lossy(&success.stderr)); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn cli_batch_rejects_bad_flags_before_creating_outputs() { + for args in [ + vec!["batch", "--steps"], + vec!["batch", "--recursive", "--root"], + vec!["batch", "--steps", "convert", "--out-dir"], + vec!["batch", "--steps", "convert", "--steps", "ocr", "--out-dir", "out"], + vec!["batch", "--unknown"], + ] { + let output = Command::new(BIN).args(args).output().unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + } +} diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index 527fe6641..60bd9eef4 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -16,6 +16,7 @@ pdfcraft-organize = { workspace = true } pdfcraft-platform = { workspace = true } serde_json = { workspace = true } sha2 = "0.11" +tempfile = "3.27" png = "0.18" base64 = { version = "0.22", optional = true } diff --git a/crates/automation/README.md b/crates/automation/README.md index 971125024..2752c6352 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -134,3 +134,9 @@ faces complete absent subset glyphs. `text_edit.ranges` accepts disjoint UTF-8 b undoable and survive saving/reopening. `image_hit` shares the UI's curve and hole hit testing. The bounds, limits and font requirements are in [existing-text editing](../../docs/editing-existing-text.md) and [image editing](../../docs/editing-images.md). + +## Folder batches + +`doc_batch` runs conversion, OCR and compression over files and optionally recursive +folders. The CLI also exposes it as `pdfcraft-cli batch`. See the +[batch guide](../../docs/cli-batch.md) for examples, output protection, errors and limits. diff --git a/crates/automation/src/a11y.rs b/crates/automation/src/a11y.rs index b10a94f45..b6a5feaf1 100644 --- a/crates/automation/src/a11y.rs +++ b/crates/automation/src/a11y.rs @@ -490,7 +490,7 @@ impl Automation { } impl Automation { - fn ocr_settings(&self, a: &Args) -> Result { + pub(crate) fn ocr_settings(&self, a: &Args) -> Result { let mut settings = pdfcraft_engine::ocr::OcrSettings::default(); if let Some(d) = a.opt_num("dpi")? { settings.dpi = d.clamp(72.0, 600.0) as f32; diff --git a/crates/automation/src/batch.rs b/crates/automation/src/batch.rs new file mode 100644 index 000000000..99310832f --- /dev/null +++ b/crates/automation/src/batch.rs @@ -0,0 +1,276 @@ +//! One-shot folder jobs shared by CLI and MCP. Discovery finishes before any output is made. + +use std::collections::{BTreeMap, HashSet}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use pdfcraft_engine::{Session, ocr, optimize}; +use serde_json::{Value, json}; + +use crate::{Args, Automation, Result, failed}; + +const MAX_ENTRIES: usize = 10_000; +const MAX_DEPTH: usize = 64; +const MAX_INPUT: u64 = 512 * 1024 * 1024; + +struct Recipe<'a> { + steps: Vec<&'a str>, + images: optimize::ImageSettings, + ocr: ocr::OcrSettings, +} + +impl<'a> Recipe<'a> { + fn parse(auto: &Automation, args: &'a Args) -> Result { + let steps = args.strs("steps")?; + if steps.is_empty() || steps.len() > 3 { + return Err(Args::wrong("steps", "one to three unique steps: convert, ocr, optimize")); + } + let mut seen = HashSet::new(); + for (i, step) in steps.iter().enumerate() { + if !matches!(*step, "convert" | "ocr" | "optimize") || !seen.insert(*step) || (*step == "convert" && i != 0) { + return Err(Args::wrong("steps", "unique convert/ocr/optimize steps, with convert first when present")); + } + } + for (keys, step) in [(&["ppi", "quality"][..], "optimize"), (&["dpi", "language", "skip_text_pages"][..], "ocr")] { + if !steps.contains(&step) && keys.iter().any(|k| args.get(k).is_some()) { + return Err(failed(format!("{} settings need the {step} step", keys.join("/")))); + } + } + let mut images = optimize::ImageSettings::REDUCE; + if let Some(ppi) = args.opt_num("ppi")? { + if !(9.0..=1600.0).contains(&ppi) { + return Err(Args::wrong("ppi", "9 to 1600")); + } + images.target_ppi = ppi; + images.above_ppi = ppi * 1.5; + } + if let Some(q) = args.opt_int("quality")? { + if !(1..=100).contains(&q) { + return Err(Args::wrong("quality", "1 to 100")); + } + images.compression = optimize::Compression::Jpeg(q as u8); + } + if args.opt_num("dpi")?.is_some_and(|dpi| !(72.0..=600.0).contains(&dpi)) { + return Err(Args::wrong("dpi", "72 to 600")); + } + Ok(Self { steps, images, ocr: auto.ocr_settings(args)? }) + } + + fn accepts(&self, path: &Path) -> bool { + let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_ascii_lowercase(); + extension == "pdf" || (self.steps.contains(&"convert") && pdfcraft_engine::CONVERTIBLE.contains(&extension.as_str())) + } + + fn process(&self, path: &Path) -> Result<(Arc>, Vec)> { + // A fresh session per file bounds retained document state and never touches caller tabs. + let mut session = Session::new(); + let mut input = Vec::new(); + std::fs::File::open(path).map_err(failed)?.take(MAX_INPUT + 1).read_to_end(&mut input).map_err(failed)?; + if input.len() as u64 > MAX_INPUT { + return Err(failed("input exceeds 512 MiB")); + } + let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| failed("input name is not UTF-8"))?; + let mut bytes = Arc::new(input); + let mut log = Vec::new(); + for step in &self.steps { + let before = bytes.len(); + match *step { + "convert" => bytes = session.convert_to_pdf(name, &bytes)?.1, + "ocr" => { + let engine = ocr::engine().map_err(failed)?; + let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &engine, |_, _| true).map_err(failed)?; + log.push( + json!({"step":"ocr","words":result.words(),"skipped_pages":result.pages.iter().filter(|p| p.skipped.is_some()).count()}), + ); + bytes = result.bytes; + continue; + } + "optimize" => { + let id = session.open(name, None, bytes.clone(), None).map_err(failed)?; + let settings = optimize::Settings { color: self.images, gray: self.images, ..Default::default() }; + let (candidate, _) = session.optimized_bytes(id, &settings, &[])?; + session.close(id); + if candidate.len() < bytes.len() { + bytes = candidate; + } + } + _ => return Err(failed("unknown batch step")), + } + log.push(json!({"step":step,"bytes_before":before,"bytes_after":bytes.len()})); + } + Ok((bytes, log)) + } +} + +// Windows junctions are reparse points too; do not descend into them as ordinary folders. +fn linked(metadata: &std::fs::Metadata) -> bool { + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + metadata.file_attributes() & 0x400 != 0 + } + #[cfg(not(windows))] + { + metadata.file_type().is_symlink() + } +} + +/// Canonicalize the existing ancestor, including when the requested output does not exist yet. +fn absolute(path: &Path) -> Result { + let mut ancestor = path; + let mut tail = Vec::new(); + while !ancestor.exists() { + if ancestor.symlink_metadata().is_ok() { + return Err(failed("broken link in batch path")); + } + tail.push(ancestor.file_name().ok_or_else(|| failed("invalid batch path"))?); + ancestor = ancestor.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); + } + let mut path = ancestor.canonicalize().map_err(failed)?; + path.extend(tail.iter().rev()); + crate::lexical(&path).ok_or_else(|| failed("invalid batch path")) +} + +struct Discovery { + files: BTreeMap, + skipped: Vec, + seen: HashSet, + visited: usize, +} + +impl Discovery { + fn walk(&mut self, path: &Path, relative: &Path, out: &Path, recursive: bool, depth: usize) -> Result<()> { + self.visited += 1; + if self.visited > MAX_ENTRIES || depth > MAX_DEPTH { + return Err(failed("batch discovery exceeds 10000 entries or 64 folder levels")); + } + if depth > 0 && path.starts_with(out) { + return Ok(()); + } + let metadata = path.symlink_metadata().map_err(|e| failed(format!("{}: {e}", path.display())))?; + if linked(&metadata) || (!metadata.is_file() && !metadata.is_dir()) { + self.skipped.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":"link or special file"})); + } else if (metadata.is_file() || depth == 0 || recursive) && self.seen.insert(path.to_path_buf()) { + if metadata.is_file() { + self.files.insert(path.to_path_buf(), relative.to_path_buf()); + } else if depth == 0 || recursive { + // Bound the list before sorting it; never collect an unbounded folder listing. + let mut entries = Vec::new(); + for entry in std::fs::read_dir(path).map_err(failed)? { + if entries.len() >= MAX_ENTRIES.saturating_sub(self.visited) { + return Err(failed("batch discovery exceeds 10000 entries")); + } + entries.push(entry.map_err(failed)?.path()); + } + entries.sort(); + for child in entries { + let name = child.file_name().ok_or_else(|| failed("entry has no name"))?; + self.walk(&child, &relative.join(name), out, recursive, depth + 1)?; + } + } + } + Ok(()) + } +} + +impl Automation { + pub(crate) fn doc_batch(&self, args: &Args) -> Result { + let recipe = Recipe::parse(self, args)?; + let recursive = args.opt_bool("recursive")?.unwrap_or(false); + let stop = args.opt_bool("stop_on_error")?.unwrap_or(false); + let paths = args.strs("paths")?; + if paths.is_empty() || paths.len() > 1000 { + return Err(Args::wrong("paths", "1 to 1000 files or folders")); + } + let out = absolute(&self.resolve(args.str("out_dir")?, true)?)?; + let mut roots = Vec::new(); + for path in paths { + let resolved = self.resolve(path, false)?; + let original = self.root.as_ref().map_or_else(|| PathBuf::from(path), |root| root.join(path)); + if linked(&original.symlink_metadata().map_err(failed)?) { + return Err(failed("input roots must be regular files or folders, not links")); + } + let real = absolute(&resolved)?; + if real.is_dir() && real.starts_with(&out) { + return Err(failed("out_dir must not contain an input folder or equal it")); + } + roots.push(real); + } + let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: 0 }; + for root in roots { + let relative = if root.is_dir() { PathBuf::new() } else { PathBuf::from(root.file_name().ok_or_else(|| failed("input has no name"))?) }; + discovery.walk(&root, &relative, &out, recursive, 0)?; + } + let mut files = discovery.skipped; + let (mut completed, mut failed_count) = (0, 0); + let mut stopped = false; + for (path, relative) in discovery.files { + if stopped || !recipe.accepts(&path) { + files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}})); + continue; + } + // Recheck confinement and links before reading a discovered file. + let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?; + if linked(&path.symlink_metadata().map_err(failed)?) || absolute(¤t)? != path { + return Err(failed("input path changed after discovery")); + } + let (bytes, log) = recipe.process(¤t)?; + let target = out.join(relative).with_extension("pdf"); + let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?; + if !target.starts_with(&out) { + return Err(failed("output path leaves out_dir")); + } + let output = publish(&target, &bytes)?; + Ok((output, bytes.len(), log)) + })) + .unwrap_or_else(|_| Err(failed("batch conversion panicked; no output was published"))); + match result { + Ok((output, bytes, steps)) => { + completed += 1; + files.push(json!({"path":path.to_string_lossy(),"status":"ok","output":output.to_string_lossy(),"bytes":bytes,"steps":steps})); + } + Err(e) => { + failed_count += 1; + files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()})); + stopped = stop; + } + } + } + let skipped = files.len().saturating_sub(completed + failed_count); + Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) + } +} + +/// Publish atomically without replacing any existing name, even if another writer wins a race. +fn publish(target: &Path, bytes: &[u8]) -> Result { + let dir = target.parent().ok_or_else(|| failed("output has no parent"))?; + std::fs::create_dir_all(dir).map_err(failed)?; + let mut temp = tempfile::NamedTempFile::new_in(dir).map_err(failed)?; + temp.write_all(bytes).map_err(failed)?; + temp.as_file().sync_all().map_err(failed)?; + let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy(); + for n in 1..=10_000 { + let candidate = if n == 1 { target.to_path_buf() } else { dir.join(format!("{stem} ({n}).pdf")) }; + match temp.persist_noclobber(&candidate) { + Ok(_) => return Ok(candidate), + Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists || candidate.symlink_metadata().is_ok() => temp = e.file, + Err(e) => return Err(failed(e.error)), + } + } + Err(failed("all 10000 output names are occupied")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_limits_fail_before_reading_another_entry() { + let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: MAX_ENTRIES }; + assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, 0).unwrap_err().to_string().contains("10000")); + discovery.visited = 0; + assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64")); + } +} diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index eb681a95b..5f5eaebb2 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -12,6 +12,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod a11y; +mod batch; mod comments; mod content; mod conventions; @@ -165,6 +166,7 @@ impl Automation { let a = Args(args); let out = match name { "doc_open" => self.doc_open(&a)?, + "doc_batch" => self.doc_batch(&a)?, "doc_list" => json!({ "documents": self.session.docs().iter().map(summary).collect::>() }), "doc_info" => { let id = self.doc(&a)?.id; diff --git a/crates/automation/src/mcp.rs b/crates/automation/src/mcp.rs index d114ea6d9..f5fda5be9 100644 --- a/crates/automation/src/mcp.rs +++ b/crates/automation/src/mcp.rs @@ -329,7 +329,7 @@ fn guarded_tool(f: impl FnOnce() -> Result) -> Result) -> Value { - let failed = name == "command_batch" + let failed = matches!(name, "command_batch" | "doc_batch") && content.iter().any(|c| match c { Content::Json(v) => v.get("failed").and_then(Value::as_u64).is_some_and(|n| n > 0), _ => false, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 6b742d261..ad407f8ab 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -118,6 +118,17 @@ pub fn tools() -> Vec { let save_out = json!({ "type": "string", "description": "File to write. Omit to open the result as a new unsaved document instead." }); let open = json!({ "type": "boolean", "description": "Also open the result as a new document (default: only when out is omitted)." }); vec![ + t("doc_batch", "Process files and folders", "Run convert, ocr and/or optimize over files and folders, one file at a time. recursive defaults to false. Preserve paths relative to each selected folder inside out_dir; duplicate inputs run once. Skip unsupported files, links and the output subtree. Existing outputs receive numbered names and are never replaced. Continue after per-file errors unless stop_on_error is true. Return completed/failed/skipped counts and per-file results. Discovery is bounded to 10000 entries and 64 folder levels. Convert must be first; each step can appear once. OCR uses installed models. Optimize uses Reduce File Size settings, optionally overriding target ppi (threshold 1.5 times ppi) and JPEG quality; a larger candidate keeps the original bytes. Sources and open documents are unchanged.") + .with(schema(json!({ + "paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000}, + "out_dir": {"type":"string"}, + "steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true}, + "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, + "ppi": {"type":"number","minimum":9,"maximum":1600}, + "quality": {"type":"integer","minimum":1,"maximum":100}, + "dpi": {"type":"number","minimum":72,"maximum":600}, + "language": {"type":"string"}, "skip_text_pages": {"type":"boolean"} + }), &["paths","out_dir","steps"])), t("doc_open", "Open a PDF", "Open a PDF file and return its document id, page count and whether it can be edited.") .cmd("file.open") .with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])), diff --git a/crates/automation/tests/batch.rs b/crates/automation/tests/batch.rs new file mode 100644 index 000000000..210187ca5 --- /dev/null +++ b/crates/automation/tests/batch.rs @@ -0,0 +1,201 @@ +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_engine::Session; +use serde_json::{Value, json}; + +fn result(auto: &mut Automation, args: Value) -> Value { + match auto.call("doc_batch", &args).unwrap().remove(0) { + Content::Json(value) => value, + _ => panic!("expected a report"), + } +} + +fn pdf() -> Vec { + Session::new().create_from_text("Original", "Original text").unwrap().as_ref().clone() +} + +#[test] +fn batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("input/output/sub")).unwrap(); + std::fs::write(dir.path().join("input/a.txt"), "Hello").unwrap(); + std::fs::write(dir.path().join("input/b.pdf"), b"broken").unwrap(); + std::fs::write(dir.path().join("input/sub/a.txt"), "World").unwrap(); + std::fs::write(dir.path().join("input/notes.bin"), b"ignore").unwrap(); + std::fs::write(dir.path().join("input/output/sub/a.pdf"), b"precious").unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let args = json!({"paths":["input","input/a.txt"],"steps":["convert","optimize"],"out_dir":"input/output","recursive":true}); + let report = result(&mut auto, args.clone()); + assert_eq!(report["completed"], 2, "{report}"); + assert_eq!(report["failed"], 1); + assert_eq!(report["skipped"], 1); + assert_eq!(std::fs::read(dir.path().join("input/output/sub/a.pdf")).unwrap(), b"precious"); + assert_eq!(std::fs::read_to_string(dir.path().join("input/a.txt")).unwrap(), "Hello"); + for name in ["a.pdf", "sub/a (2).pdf"] { + let bytes = std::fs::read(dir.path().join("input/output").join(name)).unwrap(); + let mut session = Session::new(); + let id = session.open(name, None, std::sync::Arc::new(bytes), None).unwrap(); + assert_eq!(session.get(id).unwrap().info.pages.len(), 1); + } + assert!(!dir.path().join("input/output/b.pdf").exists()); + assert!(auto.session().docs().is_empty()); + let again = result(&mut auto, args); + assert_eq!(again["completed"], 2, "previous outputs must not become new inputs: {again}"); + assert!(dir.path().join("input/output/a (2).pdf").exists()); +} + +#[test] +fn batch_is_nonrecursive_by_default_and_stop_reports_unprocessed_files() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::write(dir.path().join("input/0.pdf"), b"broken").unwrap(); + std::fs::write(dir.path().join("input/1.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/sub/2.pdf"), pdf()).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","stop_on_error":true})); + assert_eq!(report["failed"], 1); + assert_eq!(report["completed"], 0); + assert_eq!(report["skipped"], 1); + assert_eq!(report["stopped"], true); + assert!(!dir.path().join("out").exists()); + let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"})); + assert_eq!(report["completed"], 1); + assert!(!dir.path().join("out/sub").exists()); + assert!(std::fs::metadata(dir.path().join("out/1.pdf")).unwrap().len() <= pdf().len() as u64); +} + +#[test] +fn invalid_batch_arguments_and_outside_paths_write_nothing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + for change in [ + json!({"steps":["convert","convert"]}), + json!({"steps":["optimize","convert"]}), + json!({"steps":[]}), + json!({"steps":["nope"]}), + json!({"recursive":"yes"}), + json!({"paths":[]}), + json!({"paths":["input","../outside"]}), + json!({"out_dir":"../outside"}), + json!({"out_dir":"input"}), + json!({"out_dir":"."}), + json!({"quality":0}), + json!({"quality":"80"}), + json!({"ppi":2000}), + json!({"steps":["ocr"],"language":"not-a-language"}), + json!({"steps":["ocr"],"dpi":1}), + ] { + let mut args = json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"}); + args.as_object_mut().unwrap().extend(change.as_object().unwrap().clone()); + assert!(auto.call("doc_batch", &args).is_err(), "{args}"); + assert!(!dir.path().join("out").exists(), "invalid arguments must be refused before writing"); + } +} + +#[test] +fn explicitly_selected_subfolder_is_processed_without_recursive() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["convert"],"out_dir":"out"})); + assert_eq!(report["completed"], 1, "{report}"); + assert!(dir.path().join("out/page.pdf").exists()); +} + +#[test] +fn explicit_file_can_write_a_numbered_copy_beside_its_source() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"."})); + assert_eq!(report["completed"], 1, "{report}"); + assert_eq!(std::fs::read(dir.path().join("page.pdf")).unwrap(), pdf()); + assert_eq!(std::fs::read(dir.path().join("page (2).pdf")).unwrap(), pdf()); +} + +#[cfg(unix)] +#[test] +fn non_utf8_discovered_names_are_reported_without_panicking() { + use std::os::unix::ffi::OsStrExt; + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + std::fs::write(dir.path().join("input").join(std::ffi::OsStr::from_bytes(b"bad-\xff.pdf")), pdf()).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out"})); + assert_eq!(report["failed"], 1); + assert!(report["files"][0]["error"].as_str().unwrap().contains("UTF-8")); +} + +#[cfg(unix)] +#[test] +fn directory_links_are_not_followed_and_output_links_cannot_escape() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir(dir.path().join("outside")).unwrap(); + std::fs::create_dir(dir.path().join("out")).unwrap(); + std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap(); + std::os::unix::fs::symlink(dir.path().join("input"), dir.path().join("input/loop")).unwrap(); + std::os::unix::fs::symlink(dir.path().join("outside"), dir.path().join("out/sub")).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out","recursive":true})); + assert_eq!(report["skipped"], 1); + assert_eq!(report["failed"], 1); + assert_eq!(std::fs::read_dir(dir.path().join("outside")).unwrap().count(), 0); +} + +#[test] +fn batch_ocr_uses_existing_models_or_reports_the_missing_models_per_file() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["ocr","optimize"],"out_dir":"out"})); + if pdfcraft_engine::ocr::available() { + assert_eq!(report["completed"], 1, "{report}"); + assert_eq!(report["files"][0]["steps"][0]["skipped_pages"], 1); + assert!(dir.path().join("out/page.pdf").exists()); + } else { + assert_eq!(report["failed"], 1, "{report}"); + assert!(report["files"][0]["error"].as_str().unwrap().contains("model"), "{report}"); + assert!(!dir.path().join("out/page.pdf").exists()); + } +} + +#[test] +fn batch_output_collisions_are_safe_under_concurrent_writers() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap(); + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..2) + .map(|_| { + scope.spawn(|| { + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + barrier.wait(); + result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"out"})) + }) + }) + .collect(); + let reports: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert!(reports.iter().all(|r| r["completed"] == 1), "{reports:?}"); + assert_ne!(reports[0]["files"][0]["output"], reports[1]["files"][0]["output"]); + }); + for name in ["page.pdf", "page (2).pdf"] { + assert_eq!(std::fs::read(dir.path().join("out").join(name)).unwrap(), pdf()); + } + assert_eq!(std::fs::read_dir(dir.path().join("out")).unwrap().count(), 2, "no staging files remain"); +} + +#[cfg(feature = "mcp")] +#[test] +fn mcp_batch_errors_keep_the_structured_per_file_report() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("broken.pdf"), b"broken").unwrap(); + let input = json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"doc_batch","arguments":{"paths":["broken.pdf"],"steps":["optimize"],"out_dir":"out"}}}).to_string() + "\n"; + let mut out = Vec::new(); + pdfcraft_automation::mcp::McpServer::new(Automation::new().with_root(dir.path()).unwrap()).serve(input.as_bytes(), &mut out).unwrap(); + let report: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(report["result"]["isError"], true, "{report}"); + assert_eq!(report["result"]["structuredContent"]["failed"], 1); +} diff --git a/docs/cli-batch.md b/docs/cli-batch.md new file mode 100644 index 000000000..ffe0356cf --- /dev/null +++ b/docs/cli-batch.md @@ -0,0 +1,86 @@ +# Batch conversion, OCR and compression + +`pdfcraft-cli batch` runs the same steps over files or folders, one file at a time. +It writes separate PDF outputs and a JSON report to stdout. It does not change +source files or documents open in the editor. + +```sh +# Compress all PDFs, including subfolders. +pdfcraft-cli batch scans --root . --recursive --steps optimize --out-dir compressed + +# Convert images and text, then compress. Existing PDFs are accepted too. +pdfcraft-cli batch inputs --root . --recursive --steps convert,optimize --out-dir pdfs --ppi 150 --quality 80 + +# Add searchable text before compression. Requires the installed OCR models. +pdfcraft-cli batch scans --root . --recursive --steps ocr,optimize --out-dir searchable +``` + +Multiple input files or folders can be listed before the options. Directories are +read only one level deep unless `--recursive` is present. Paths within each input +folder are retained inside the output folder. Files encountered more than once +through overlapping selections run once; the first selection determines their +relative output path. Processing order is sorted by input path. + +Steps are `convert`, `ocr` and `optimize`, each at most once. `convert`, when used, +must come first. Conversion uses the same supported extensions as Create PDF; +new engine importers become available here automatically. Without `convert`, +only `.pdf` inputs are processed. Unsupported extensions are reported as skipped. + +- `optimize` uses Reduce File Size settings: JPEG quality 60, resampling to 150 ppi + above 225 ppi, plus the existing document cleanup. `--ppi` (9–1600) sets the target + and a threshold of 1.5 times that value; `--quality` (1–100) sets JPEG quality. + These settings can lose image detail. A candidate that is not smaller retains + the bytes entering that step. This is not an images-only or lossless preset. +- `ocr` uses the installed engine and its supported languages. `--dpi` (72–600, + default 300), `--language` (default `en`) and `--ocr-all-pages` configure it. + By default pages already containing text are skipped. Missing models are + reported as file errors; nothing is downloaded or installed by the batch. +- `--stop-on-error` stops processing after the first file error. By default the + remaining files continue. Earlier successful outputs stay, and unprocessed + files are listed as skipped. A file is published only after all its steps succeed. + +Existing output names are never replaced, even by concurrent batch writers: +`book.pdf`, `book (2).pdf`, and so on. A temporary file in the destination folder +is published without overwriting. Sources are never moved or deleted. An output +folder inside an input folder is excluded from discovery, including on reruns. +The output folder cannot equal or contain a selected input folder. + +The report contains `completed`, `failed`, `skipped`, `stopped` and `files`. +Each file records its status, output and step results, or the error/skip reason. +The `batch` command exits nonzero on any file failure while still printing the +report. Redirect stdout to retain it; keep the report outside the input folders. + +## MCP and JSON scripts + +The same implementation is available as `doc_batch`: + +```json +{ + "paths": ["inputs"], + "steps": ["convert", "optimize"], + "recursive": true, + "out_dir": "pdfs", + "ppi": 150, + "quality": 80, + "stop_on_error": false +} +``` + +In `pdfcraft-cli run --script`, put this object in the `args` of a `doc_batch` +step. MCP marks a report with failed files as `isError: true` and retains the +structured report. Tool names and OCR settings match the existing automation API +(`skip_text_pages: false` corresponds to `--ocr-all-pages`). + +## Boundaries + +Discovery completes before writing. Invalid arguments, unreadable input folders +or paths outside `--root` refuse the call. Traversal accepts at most 1,000 selected +paths, 10,000 visited entries and 64 folder levels. Discovered symbolic links, +Windows reparse points and special files are skipped; selecting a link as a root +is refused. Individual input files are limited to 512 MiB, in addition to engine +limits. Signed documents cannot be optimized, and encrypted PDFs requiring a +password are not supported by this command. + +This is a synchronous, one-shot job. It has no folder watcher, scheduler, resume +database, concurrent file processing or GUI progress dialog. Each file has its +own temporary engine session; completed documents are not retained in memory. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 4da4fbcaf..e36d1420a 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -9287,9 +9287,9 @@ tier = "P1" milestone = "M13" status = "partial" commands = ["actions.wizard", "ocr.recognize_batch"] -tools = ["action_run", "ocr_recognize_files"] -evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions"] -notes = "Actions and OCR run over chosen files; whole folders (with subfolders) not yet." +tools = ["action_run", "ocr_recognize_files", "doc_batch"] +evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions", "crates/automation/tests/batch.rs::batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs", "apps/pdfcraft-cli/tests/batch.rs::cli_batch_reports_partial_failure_and_continues_into_subfolders"] +notes = "Actions and OCR run over chosen files. CLI/MCP doc_batch adds bounded recursive folders for convert/ocr/optimize recipes, retained relative paths, collision-safe outputs and per-file reports. GUI folder selection, generic action recipes, scheduling and resume remain absent. See docs/cli-batch.md." [[feature]] id = "misc.action-prompt-steps"