M13: Run conversion, OCR and optimization over folder trees (#1605)

Add a shared CLI/MCP batch runner with bounded discovery, retained relative
paths and per-file processing in isolated engine sessions. Publish complete
outputs without overwriting existing names, and return structured errors
with an optional stop-on-error policy.

Cover recursive jobs, overlapping inputs, concurrent output collisions,
root confinement and partial-failure reporting in automation and CLI tests.

Co-authored-by: LabDennis <341377186+LabDennis@users.noreply.github.com>
This commit is contained in:
Sir Clangington III.
2026-10-13 00:12:28 -04:00
committed by GitHub
co-authored by LabDennis
parent 73efaef13e
commit 7a0f846a28
14 changed files with 702 additions and 5 deletions
Generated
+1
View File
@@ -3632,6 +3632,7 @@ dependencies = [
"png 0.18.1",
"serde_json",
"sha2",
"tempfile",
]
[[package]]
+61
View File
@@ -0,0 +1,61 @@
//! Friendly flags for the same folder-job tool exposed by `run` and MCP.
use serde_json::json;
use crate::{CliError, OptionSpec, automation, flag, print_output, validate_options};
const OPTIONS: &[OptionSpec] = &[
OptionSpec { name: "--root", takes_value: true },
OptionSpec { name: "--out-dir", takes_value: true },
OptionSpec { name: "--steps", takes_value: true },
OptionSpec { name: "--recursive", takes_value: false },
OptionSpec { name: "--stop-on-error", takes_value: false },
OptionSpec { name: "--ppi", takes_value: true },
OptionSpec { name: "--quality", takes_value: true },
OptionSpec { name: "--dpi", takes_value: true },
OptionSpec { name: "--language", takes_value: true },
OptionSpec { name: "--ocr-all-pages", takes_value: false },
];
pub(super) fn run(args: &[String]) -> Result<(), CliError> {
validate_options("batch", args, OPTIONS)?;
let mut paths = Vec::new();
let mut seen = std::collections::HashSet::new();
let mut i = 0;
while let Some(arg) = args.get(i) {
if let Some(spec) = OPTIONS.iter().find(|s| s.name == arg) {
if !seen.insert(spec.name) {
return Err(format!("batch: {} may only be supplied once", spec.name).into());
}
if spec.takes_value {
if args.get(i + 1).is_none_or(|v| v.is_empty() || v.starts_with("--")) {
return Err(format!("batch: {} needs a value", spec.name).into());
}
i += 1;
}
} else {
paths.push(arg);
}
i += 1;
}
let steps: Vec<_> = flag(args, "--steps").ok_or("batch: missing --steps (convert, ocr, optimize)")?.split(',').collect();
let out = flag(args, "--out-dir").ok_or("batch: missing --out-dir")?;
let mut params =
json!({"paths":paths,"steps":steps,"out_dir":out,"recursive":seen.contains("--recursive"),"stop_on_error":seen.contains("--stop-on-error")});
for (option, key) in [("--ppi", "ppi"), ("--quality", "quality"), ("--dpi", "dpi")] {
if let Some(v) = flag(args, option) {
params[key] = serde_json::from_str(v).map_err(|_| format!("batch: {option} needs a number"))?;
}
}
if let Some(lang) = flag(args, "--language") {
params["language"] = json!(lang);
}
if seen.contains("--ocr-all-pages") {
params["skip_text_pages"] = json!(false);
}
let mut auto = automation(args)?;
let content = auto.call("doc_batch", &params).map_err(|e| e.to_string())?;
let failed = content.iter().any(|c| matches!(c, pdfcraft_automation::Content::Json(v) if v["failed"].as_u64().is_some_and(|n| n > 0)));
print_output(&auto, content, None)?;
if failed { Err("batch: one or more files failed; see the JSON report".into()) } else { Ok(()) }
}
+6
View File
@@ -17,6 +17,8 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod batch;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode, Stdio};
@@ -57,6 +59,9 @@ pdfcraft-cli extract <in.pdf> --pages 1,3,5 --out out.pdf
pdfcraft-cli split <in.pdf> (--every N | --before 3,7) [--out-dir DIR]
pdfcraft-cli check <files or dirs…> [--timeout 20] [--dpi 36] [--json out.json]
pdfcraft-cli audit <file.pdf> [--password PW] find faux redactions (fails when found)
pdfcraft-cli batch <files or dirs...> --steps convert,ocr,optimize --out-dir DIR
[--recursive] [--root DIR] [--stop-on-error]
[--ppi 150] [--quality 60] [--dpi 300] [--language en] [--ocr-all-pages]
pdfcraft-cli tools automation tools and their JSON Schemas
pdfcraft-cli run <tool> [key=value …] [--root DIR] [--out image.png]
pdfcraft-cli run --script steps.json [--root DIR] [{\"tool\": \"doc_open\", \"args\": {…}}, …]
@@ -90,6 +95,7 @@ fn main() -> ExitCode {
Some("audit") => audit(&args[1..]),
Some("tools") => tools(),
Some("run") => run(&args[1..]),
Some("batch") => batch::run(&args[1..]),
Some("ui") => ui(&args[1..]),
#[cfg(feature = "mcp")]
Some("mcp") => mcp(&args[1..]),
+46
View File
@@ -0,0 +1,46 @@
use serde_json::Value;
use std::process::Command;
const BIN: &str = env!("CARGO_BIN_EXE_pdfcraft-cli");
#[test]
fn cli_batch_reports_partial_failure_and_continues_into_subfolders() {
let dir = std::env::temp_dir().join(format!("pdfcraft-cli-batch-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("input/sub")).unwrap();
std::fs::write(dir.join("input/0.pdf"), b"broken").unwrap();
std::fs::write(dir.join("input/sub/page.txt"), "Batch text").unwrap();
let output = Command::new(BIN)
.args(["batch", "input", "--root"])
.arg(&dir)
.args(["--steps", "convert,optimize", "--out-dir", "out", "--recursive"])
.output()
.unwrap();
assert!(!output.status.success());
let report: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(report["completed"], 1, "{report}");
assert_eq!(report["failed"], 1);
let converted = dir.join("out/sub/page.pdf");
assert!(converted.exists());
let text = Command::new(BIN).arg("text").arg(converted).output().unwrap();
assert!(text.status.success());
assert!(String::from_utf8_lossy(&text.stdout).contains("Batch text"));
let success = Command::new(BIN).current_dir(&dir).args(["batch", "input/sub", "--steps", "convert", "--out-dir", "other"]).output().unwrap();
assert!(success.status.success(), "{}", String::from_utf8_lossy(&success.stderr));
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn cli_batch_rejects_bad_flags_before_creating_outputs() {
for args in [
vec!["batch", "--steps"],
vec!["batch", "--recursive", "--root"],
vec!["batch", "--steps", "convert", "--out-dir"],
vec!["batch", "--steps", "convert", "--steps", "ocr", "--out-dir", "out"],
vec!["batch", "--unknown"],
] {
let output = Command::new(BIN).args(args).output().unwrap();
assert!(!output.status.success());
assert!(output.stdout.is_empty());
}
}
+1
View File
@@ -16,6 +16,7 @@ pdfcraft-organize = { workspace = true }
pdfcraft-platform = { workspace = true }
serde_json = { workspace = true }
sha2 = "0.11"
tempfile = "3.27"
png = "0.18"
base64 = { version = "0.22", optional = true }
+6
View File
@@ -134,3 +134,9 @@ faces complete absent subset glyphs. `text_edit.ranges` accepts disjoint UTF-8 b
undoable and survive saving/reopening. `image_hit` shares the UI's curve and hole hit testing.
The bounds, limits and font requirements are in [existing-text editing](../../docs/editing-existing-text.md)
and [image editing](../../docs/editing-images.md).
## Folder batches
`doc_batch` runs conversion, OCR and compression over files and optionally recursive
folders. The CLI also exposes it as `pdfcraft-cli batch`. See the
[batch guide](../../docs/cli-batch.md) for examples, output protection, errors and limits.
+1 -1
View File
@@ -490,7 +490,7 @@ impl Automation {
}
impl Automation {
fn ocr_settings(&self, a: &Args) -> Result<pdfcraft_engine::ocr::OcrSettings> {
pub(crate) fn ocr_settings(&self, a: &Args) -> Result<pdfcraft_engine::ocr::OcrSettings> {
let mut settings = pdfcraft_engine::ocr::OcrSettings::default();
if let Some(d) = a.opt_num("dpi")? {
settings.dpi = d.clamp(72.0, 600.0) as f32;
+276
View File
@@ -0,0 +1,276 @@
//! One-shot folder jobs shared by CLI and MCP. Discovery finishes before any output is made.
use std::collections::{BTreeMap, HashSet};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use pdfcraft_engine::{Session, ocr, optimize};
use serde_json::{Value, json};
use crate::{Args, Automation, Result, failed};
const MAX_ENTRIES: usize = 10_000;
const MAX_DEPTH: usize = 64;
const MAX_INPUT: u64 = 512 * 1024 * 1024;
struct Recipe<'a> {
steps: Vec<&'a str>,
images: optimize::ImageSettings,
ocr: ocr::OcrSettings,
}
impl<'a> Recipe<'a> {
fn parse(auto: &Automation, args: &'a Args) -> Result<Self> {
let steps = args.strs("steps")?;
if steps.is_empty() || steps.len() > 3 {
return Err(Args::wrong("steps", "one to three unique steps: convert, ocr, optimize"));
}
let mut seen = HashSet::new();
for (i, step) in steps.iter().enumerate() {
if !matches!(*step, "convert" | "ocr" | "optimize") || !seen.insert(*step) || (*step == "convert" && i != 0) {
return Err(Args::wrong("steps", "unique convert/ocr/optimize steps, with convert first when present"));
}
}
for (keys, step) in [(&["ppi", "quality"][..], "optimize"), (&["dpi", "language", "skip_text_pages"][..], "ocr")] {
if !steps.contains(&step) && keys.iter().any(|k| args.get(k).is_some()) {
return Err(failed(format!("{} settings need the {step} step", keys.join("/"))));
}
}
let mut images = optimize::ImageSettings::REDUCE;
if let Some(ppi) = args.opt_num("ppi")? {
if !(9.0..=1600.0).contains(&ppi) {
return Err(Args::wrong("ppi", "9 to 1600"));
}
images.target_ppi = ppi;
images.above_ppi = ppi * 1.5;
}
if let Some(q) = args.opt_int("quality")? {
if !(1..=100).contains(&q) {
return Err(Args::wrong("quality", "1 to 100"));
}
images.compression = optimize::Compression::Jpeg(q as u8);
}
if args.opt_num("dpi")?.is_some_and(|dpi| !(72.0..=600.0).contains(&dpi)) {
return Err(Args::wrong("dpi", "72 to 600"));
}
Ok(Self { steps, images, ocr: auto.ocr_settings(args)? })
}
fn accepts(&self, path: &Path) -> bool {
let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_ascii_lowercase();
extension == "pdf" || (self.steps.contains(&"convert") && pdfcraft_engine::CONVERTIBLE.contains(&extension.as_str()))
}
fn process(&self, path: &Path) -> Result<(Arc<Vec<u8>>, Vec<Value>)> {
// A fresh session per file bounds retained document state and never touches caller tabs.
let mut session = Session::new();
let mut input = Vec::new();
std::fs::File::open(path).map_err(failed)?.take(MAX_INPUT + 1).read_to_end(&mut input).map_err(failed)?;
if input.len() as u64 > MAX_INPUT {
return Err(failed("input exceeds 512 MiB"));
}
let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| failed("input name is not UTF-8"))?;
let mut bytes = Arc::new(input);
let mut log = Vec::new();
for step in &self.steps {
let before = bytes.len();
match *step {
"convert" => bytes = session.convert_to_pdf(name, &bytes)?.1,
"ocr" => {
let engine = ocr::engine().map_err(failed)?;
let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &engine, |_, _| true).map_err(failed)?;
log.push(
json!({"step":"ocr","words":result.words(),"skipped_pages":result.pages.iter().filter(|p| p.skipped.is_some()).count()}),
);
bytes = result.bytes;
continue;
}
"optimize" => {
let id = session.open(name, None, bytes.clone(), None).map_err(failed)?;
let settings = optimize::Settings { color: self.images, gray: self.images, ..Default::default() };
let (candidate, _) = session.optimized_bytes(id, &settings, &[])?;
session.close(id);
if candidate.len() < bytes.len() {
bytes = candidate;
}
}
_ => return Err(failed("unknown batch step")),
}
log.push(json!({"step":step,"bytes_before":before,"bytes_after":bytes.len()}));
}
Ok((bytes, log))
}
}
// Windows junctions are reparse points too; do not descend into them as ordinary folders.
fn linked(metadata: &std::fs::Metadata) -> bool {
#[cfg(windows)]
{
use std::os::windows::fs::MetadataExt;
metadata.file_attributes() & 0x400 != 0
}
#[cfg(not(windows))]
{
metadata.file_type().is_symlink()
}
}
/// Canonicalize the existing ancestor, including when the requested output does not exist yet.
fn absolute(path: &Path) -> Result<PathBuf> {
let mut ancestor = path;
let mut tail = Vec::new();
while !ancestor.exists() {
if ancestor.symlink_metadata().is_ok() {
return Err(failed("broken link in batch path"));
}
tail.push(ancestor.file_name().ok_or_else(|| failed("invalid batch path"))?);
ancestor = ancestor.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
}
let mut path = ancestor.canonicalize().map_err(failed)?;
path.extend(tail.iter().rev());
crate::lexical(&path).ok_or_else(|| failed("invalid batch path"))
}
struct Discovery {
files: BTreeMap<PathBuf, PathBuf>,
skipped: Vec<Value>,
seen: HashSet<PathBuf>,
visited: usize,
}
impl Discovery {
fn walk(&mut self, path: &Path, relative: &Path, out: &Path, recursive: bool, depth: usize) -> Result<()> {
self.visited += 1;
if self.visited > MAX_ENTRIES || depth > MAX_DEPTH {
return Err(failed("batch discovery exceeds 10000 entries or 64 folder levels"));
}
if depth > 0 && path.starts_with(out) {
return Ok(());
}
let metadata = path.symlink_metadata().map_err(|e| failed(format!("{}: {e}", path.display())))?;
if linked(&metadata) || (!metadata.is_file() && !metadata.is_dir()) {
self.skipped.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":"link or special file"}));
} else if (metadata.is_file() || depth == 0 || recursive) && self.seen.insert(path.to_path_buf()) {
if metadata.is_file() {
self.files.insert(path.to_path_buf(), relative.to_path_buf());
} else if depth == 0 || recursive {
// Bound the list before sorting it; never collect an unbounded folder listing.
let mut entries = Vec::new();
for entry in std::fs::read_dir(path).map_err(failed)? {
if entries.len() >= MAX_ENTRIES.saturating_sub(self.visited) {
return Err(failed("batch discovery exceeds 10000 entries"));
}
entries.push(entry.map_err(failed)?.path());
}
entries.sort();
for child in entries {
let name = child.file_name().ok_or_else(|| failed("entry has no name"))?;
self.walk(&child, &relative.join(name), out, recursive, depth + 1)?;
}
}
}
Ok(())
}
}
impl Automation {
pub(crate) fn doc_batch(&self, args: &Args) -> Result<Value> {
let recipe = Recipe::parse(self, args)?;
let recursive = args.opt_bool("recursive")?.unwrap_or(false);
let stop = args.opt_bool("stop_on_error")?.unwrap_or(false);
let paths = args.strs("paths")?;
if paths.is_empty() || paths.len() > 1000 {
return Err(Args::wrong("paths", "1 to 1000 files or folders"));
}
let out = absolute(&self.resolve(args.str("out_dir")?, true)?)?;
let mut roots = Vec::new();
for path in paths {
let resolved = self.resolve(path, false)?;
let original = self.root.as_ref().map_or_else(|| PathBuf::from(path), |root| root.join(path));
if linked(&original.symlink_metadata().map_err(failed)?) {
return Err(failed("input roots must be regular files or folders, not links"));
}
let real = absolute(&resolved)?;
if real.is_dir() && real.starts_with(&out) {
return Err(failed("out_dir must not contain an input folder or equal it"));
}
roots.push(real);
}
let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: 0 };
for root in roots {
let relative = if root.is_dir() { PathBuf::new() } else { PathBuf::from(root.file_name().ok_or_else(|| failed("input has no name"))?) };
discovery.walk(&root, &relative, &out, recursive, 0)?;
}
let mut files = discovery.skipped;
let (mut completed, mut failed_count) = (0, 0);
let mut stopped = false;
for (path, relative) in discovery.files {
if stopped || !recipe.accepts(&path) {
files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}}));
continue;
}
// Recheck confinement and links before reading a discovered file.
let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?;
if linked(&path.symlink_metadata().map_err(failed)?) || absolute(&current)? != path {
return Err(failed("input path changed after discovery"));
}
let (bytes, log) = recipe.process(&current)?;
let target = out.join(relative).with_extension("pdf");
let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?;
if !target.starts_with(&out) {
return Err(failed("output path leaves out_dir"));
}
let output = publish(&target, &bytes)?;
Ok((output, bytes.len(), log))
}))
.unwrap_or_else(|_| Err(failed("batch conversion panicked; no output was published")));
match result {
Ok((output, bytes, steps)) => {
completed += 1;
files.push(json!({"path":path.to_string_lossy(),"status":"ok","output":output.to_string_lossy(),"bytes":bytes,"steps":steps}));
}
Err(e) => {
failed_count += 1;
files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()}));
stopped = stop;
}
}
}
let skipped = files.len().saturating_sub(completed + failed_count);
Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files}))
}
}
/// Publish atomically without replacing any existing name, even if another writer wins a race.
fn publish(target: &Path, bytes: &[u8]) -> Result<PathBuf> {
let dir = target.parent().ok_or_else(|| failed("output has no parent"))?;
std::fs::create_dir_all(dir).map_err(failed)?;
let mut temp = tempfile::NamedTempFile::new_in(dir).map_err(failed)?;
temp.write_all(bytes).map_err(failed)?;
temp.as_file().sync_all().map_err(failed)?;
let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy();
for n in 1..=10_000 {
let candidate = if n == 1 { target.to_path_buf() } else { dir.join(format!("{stem} ({n}).pdf")) };
match temp.persist_noclobber(&candidate) {
Ok(_) => return Ok(candidate),
Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists || candidate.symlink_metadata().is_ok() => temp = e.file,
Err(e) => return Err(failed(e.error)),
}
}
Err(failed("all 10000 output names are occupied"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn discovery_limits_fail_before_reading_another_entry() {
let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: MAX_ENTRIES };
assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, 0).unwrap_err().to_string().contains("10000"));
discovery.visited = 0;
assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64"));
}
}
+2
View File
@@ -12,6 +12,7 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod a11y;
mod batch;
mod comments;
mod content;
mod conventions;
@@ -165,6 +166,7 @@ impl Automation {
let a = Args(args);
let out = match name {
"doc_open" => self.doc_open(&a)?,
"doc_batch" => self.doc_batch(&a)?,
"doc_list" => json!({ "documents": self.session.docs().iter().map(summary).collect::<Vec<_>>() }),
"doc_info" => {
let id = self.doc(&a)?.id;
+1 -1
View File
@@ -329,7 +329,7 @@ fn guarded_tool(f: impl FnOnce() -> Result<Value, (i64, String)>) -> Result<Valu
}
fn tool_content(name: &str, content: Vec<Content>) -> Value {
let failed = name == "command_batch"
let failed = matches!(name, "command_batch" | "doc_batch")
&& content.iter().any(|c| match c {
Content::Json(v) => v.get("failed").and_then(Value::as_u64).is_some_and(|n| n > 0),
_ => false,
+11
View File
@@ -118,6 +118,17 @@ pub fn tools() -> Vec<ToolDef> {
let save_out = json!({ "type": "string", "description": "File to write. Omit to open the result as a new unsaved document instead." });
let open = json!({ "type": "boolean", "description": "Also open the result as a new document (default: only when out is omitted)." });
vec![
t("doc_batch", "Process files and folders", "Run convert, ocr and/or optimize over files and folders, one file at a time. recursive defaults to false. Preserve paths relative to each selected folder inside out_dir; duplicate inputs run once. Skip unsupported files, links and the output subtree. Existing outputs receive numbered names and are never replaced. Continue after per-file errors unless stop_on_error is true. Return completed/failed/skipped counts and per-file results. Discovery is bounded to 10000 entries and 64 folder levels. Convert must be first; each step can appear once. OCR uses installed models. Optimize uses Reduce File Size settings, optionally overriding target ppi (threshold 1.5 times ppi) and JPEG quality; a larger candidate keeps the original bytes. Sources and open documents are unchanged.")
.with(schema(json!({
"paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000},
"out_dir": {"type":"string"},
"steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true},
"recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"},
"ppi": {"type":"number","minimum":9,"maximum":1600},
"quality": {"type":"integer","minimum":1,"maximum":100},
"dpi": {"type":"number","minimum":72,"maximum":600},
"language": {"type":"string"}, "skip_text_pages": {"type":"boolean"}
}), &["paths","out_dir","steps"])),
t("doc_open", "Open a PDF", "Open a PDF file and return its document id, page count and whether it can be edited.")
.cmd("file.open")
.with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])),
+201
View File
@@ -0,0 +1,201 @@
use pdfcraft_automation::{Automation, Content};
use pdfcraft_engine::Session;
use serde_json::{Value, json};
fn result(auto: &mut Automation, args: Value) -> Value {
match auto.call("doc_batch", &args).unwrap().remove(0) {
Content::Json(value) => value,
_ => panic!("expected a report"),
}
}
fn pdf() -> Vec<u8> {
Session::new().create_from_text("Original", "Original text").unwrap().as_ref().clone()
}
#[test]
fn batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
std::fs::create_dir_all(dir.path().join("input/output/sub")).unwrap();
std::fs::write(dir.path().join("input/a.txt"), "Hello").unwrap();
std::fs::write(dir.path().join("input/b.pdf"), b"broken").unwrap();
std::fs::write(dir.path().join("input/sub/a.txt"), "World").unwrap();
std::fs::write(dir.path().join("input/notes.bin"), b"ignore").unwrap();
std::fs::write(dir.path().join("input/output/sub/a.pdf"), b"precious").unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let args = json!({"paths":["input","input/a.txt"],"steps":["convert","optimize"],"out_dir":"input/output","recursive":true});
let report = result(&mut auto, args.clone());
assert_eq!(report["completed"], 2, "{report}");
assert_eq!(report["failed"], 1);
assert_eq!(report["skipped"], 1);
assert_eq!(std::fs::read(dir.path().join("input/output/sub/a.pdf")).unwrap(), b"precious");
assert_eq!(std::fs::read_to_string(dir.path().join("input/a.txt")).unwrap(), "Hello");
for name in ["a.pdf", "sub/a (2).pdf"] {
let bytes = std::fs::read(dir.path().join("input/output").join(name)).unwrap();
let mut session = Session::new();
let id = session.open(name, None, std::sync::Arc::new(bytes), None).unwrap();
assert_eq!(session.get(id).unwrap().info.pages.len(), 1);
}
assert!(!dir.path().join("input/output/b.pdf").exists());
assert!(auto.session().docs().is_empty());
let again = result(&mut auto, args);
assert_eq!(again["completed"], 2, "previous outputs must not become new inputs: {again}");
assert!(dir.path().join("input/output/a (2).pdf").exists());
}
#[test]
fn batch_is_nonrecursive_by_default_and_stop_reports_unprocessed_files() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
std::fs::write(dir.path().join("input/0.pdf"), b"broken").unwrap();
std::fs::write(dir.path().join("input/1.pdf"), pdf()).unwrap();
std::fs::write(dir.path().join("input/sub/2.pdf"), pdf()).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","stop_on_error":true}));
assert_eq!(report["failed"], 1);
assert_eq!(report["completed"], 0);
assert_eq!(report["skipped"], 1);
assert_eq!(report["stopped"], true);
assert!(!dir.path().join("out").exists());
let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"}));
assert_eq!(report["completed"], 1);
assert!(!dir.path().join("out/sub").exists());
assert!(std::fs::metadata(dir.path().join("out/1.pdf")).unwrap().len() <= pdf().len() as u64);
}
#[test]
fn invalid_batch_arguments_and_outside_paths_write_nothing() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir(dir.path().join("input")).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
for change in [
json!({"steps":["convert","convert"]}),
json!({"steps":["optimize","convert"]}),
json!({"steps":[]}),
json!({"steps":["nope"]}),
json!({"recursive":"yes"}),
json!({"paths":[]}),
json!({"paths":["input","../outside"]}),
json!({"out_dir":"../outside"}),
json!({"out_dir":"input"}),
json!({"out_dir":"."}),
json!({"quality":0}),
json!({"quality":"80"}),
json!({"ppi":2000}),
json!({"steps":["ocr"],"language":"not-a-language"}),
json!({"steps":["ocr"],"dpi":1}),
] {
let mut args = json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"});
args.as_object_mut().unwrap().extend(change.as_object().unwrap().clone());
assert!(auto.call("doc_batch", &args).is_err(), "{args}");
assert!(!dir.path().join("out").exists(), "invalid arguments must be refused before writing");
}
}
#[test]
fn explicitly_selected_subfolder_is_processed_without_recursive() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["convert"],"out_dir":"out"}));
assert_eq!(report["completed"], 1, "{report}");
assert!(dir.path().join("out/page.pdf").exists());
}
#[test]
fn explicit_file_can_write_a_numbered_copy_beside_its_source() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"."}));
assert_eq!(report["completed"], 1, "{report}");
assert_eq!(std::fs::read(dir.path().join("page.pdf")).unwrap(), pdf());
assert_eq!(std::fs::read(dir.path().join("page (2).pdf")).unwrap(), pdf());
}
#[cfg(unix)]
#[test]
fn non_utf8_discovered_names_are_reported_without_panicking() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir(dir.path().join("input")).unwrap();
std::fs::write(dir.path().join("input").join(std::ffi::OsStr::from_bytes(b"bad-\xff.pdf")), pdf()).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out"}));
assert_eq!(report["failed"], 1);
assert!(report["files"][0]["error"].as_str().unwrap().contains("UTF-8"));
}
#[cfg(unix)]
#[test]
fn directory_links_are_not_followed_and_output_links_cannot_escape() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
std::fs::create_dir(dir.path().join("outside")).unwrap();
std::fs::create_dir(dir.path().join("out")).unwrap();
std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap();
std::os::unix::fs::symlink(dir.path().join("input"), dir.path().join("input/loop")).unwrap();
std::os::unix::fs::symlink(dir.path().join("outside"), dir.path().join("out/sub")).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out","recursive":true}));
assert_eq!(report["skipped"], 1);
assert_eq!(report["failed"], 1);
assert_eq!(std::fs::read_dir(dir.path().join("outside")).unwrap().count(), 0);
}
#[test]
fn batch_ocr_uses_existing_models_or_reports_the_missing_models_per_file() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
let mut auto = Automation::new().with_root(dir.path()).unwrap();
let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["ocr","optimize"],"out_dir":"out"}));
if pdfcraft_engine::ocr::available() {
assert_eq!(report["completed"], 1, "{report}");
assert_eq!(report["files"][0]["steps"][0]["skipped_pages"], 1);
assert!(dir.path().join("out/page.pdf").exists());
} else {
assert_eq!(report["failed"], 1, "{report}");
assert!(report["files"][0]["error"].as_str().unwrap().contains("model"), "{report}");
assert!(!dir.path().join("out/page.pdf").exists());
}
}
#[test]
fn batch_output_collisions_are_safe_under_concurrent_writers() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
let barrier = std::sync::Barrier::new(2);
std::thread::scope(|scope| {
let handles: Vec<_> = (0..2)
.map(|_| {
scope.spawn(|| {
let mut auto = Automation::new().with_root(dir.path()).unwrap();
barrier.wait();
result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"out"}))
})
})
.collect();
let reports: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
assert!(reports.iter().all(|r| r["completed"] == 1), "{reports:?}");
assert_ne!(reports[0]["files"][0]["output"], reports[1]["files"][0]["output"]);
});
for name in ["page.pdf", "page (2).pdf"] {
assert_eq!(std::fs::read(dir.path().join("out").join(name)).unwrap(), pdf());
}
assert_eq!(std::fs::read_dir(dir.path().join("out")).unwrap().count(), 2, "no staging files remain");
}
#[cfg(feature = "mcp")]
#[test]
fn mcp_batch_errors_keep_the_structured_per_file_report() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("broken.pdf"), b"broken").unwrap();
let input = json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"doc_batch","arguments":{"paths":["broken.pdf"],"steps":["optimize"],"out_dir":"out"}}}).to_string() + "\n";
let mut out = Vec::new();
pdfcraft_automation::mcp::McpServer::new(Automation::new().with_root(dir.path()).unwrap()).serve(input.as_bytes(), &mut out).unwrap();
let report: Value = serde_json::from_slice(&out).unwrap();
assert_eq!(report["result"]["isError"], true, "{report}");
assert_eq!(report["result"]["structuredContent"]["failed"], 1);
}
+86
View File
@@ -0,0 +1,86 @@
# Batch conversion, OCR and compression
`pdfcraft-cli batch` runs the same steps over files or folders, one file at a time.
It writes separate PDF outputs and a JSON report to stdout. It does not change
source files or documents open in the editor.
```sh
# Compress all PDFs, including subfolders.
pdfcraft-cli batch scans --root . --recursive --steps optimize --out-dir compressed
# Convert images and text, then compress. Existing PDFs are accepted too.
pdfcraft-cli batch inputs --root . --recursive --steps convert,optimize --out-dir pdfs --ppi 150 --quality 80
# Add searchable text before compression. Requires the installed OCR models.
pdfcraft-cli batch scans --root . --recursive --steps ocr,optimize --out-dir searchable
```
Multiple input files or folders can be listed before the options. Directories are
read only one level deep unless `--recursive` is present. Paths within each input
folder are retained inside the output folder. Files encountered more than once
through overlapping selections run once; the first selection determines their
relative output path. Processing order is sorted by input path.
Steps are `convert`, `ocr` and `optimize`, each at most once. `convert`, when used,
must come first. Conversion uses the same supported extensions as Create PDF;
new engine importers become available here automatically. Without `convert`,
only `.pdf` inputs are processed. Unsupported extensions are reported as skipped.
- `optimize` uses Reduce File Size settings: JPEG quality 60, resampling to 150 ppi
above 225 ppi, plus the existing document cleanup. `--ppi` (9–1600) sets the target
and a threshold of 1.5 times that value; `--quality` (1–100) sets JPEG quality.
These settings can lose image detail. A candidate that is not smaller retains
the bytes entering that step. This is not an images-only or lossless preset.
- `ocr` uses the installed engine and its supported languages. `--dpi` (72–600,
default 300), `--language` (default `en`) and `--ocr-all-pages` configure it.
By default pages already containing text are skipped. Missing models are
reported as file errors; nothing is downloaded or installed by the batch.
- `--stop-on-error` stops processing after the first file error. By default the
remaining files continue. Earlier successful outputs stay, and unprocessed
files are listed as skipped. A file is published only after all its steps succeed.
Existing output names are never replaced, even by concurrent batch writers:
`book.pdf`, `book (2).pdf`, and so on. A temporary file in the destination folder
is published without overwriting. Sources are never moved or deleted. An output
folder inside an input folder is excluded from discovery, including on reruns.
The output folder cannot equal or contain a selected input folder.
The report contains `completed`, `failed`, `skipped`, `stopped` and `files`.
Each file records its status, output and step results, or the error/skip reason.
The `batch` command exits nonzero on any file failure while still printing the
report. Redirect stdout to retain it; keep the report outside the input folders.
## MCP and JSON scripts
The same implementation is available as `doc_batch`:
```json
{
"paths": ["inputs"],
"steps": ["convert", "optimize"],
"recursive": true,
"out_dir": "pdfs",
"ppi": 150,
"quality": 80,
"stop_on_error": false
}
```
In `pdfcraft-cli run --script`, put this object in the `args` of a `doc_batch`
step. MCP marks a report with failed files as `isError: true` and retains the
structured report. Tool names and OCR settings match the existing automation API
(`skip_text_pages: false` corresponds to `--ocr-all-pages`).
## Boundaries
Discovery completes before writing. Invalid arguments, unreadable input folders
or paths outside `--root` refuse the call. Traversal accepts at most 1,000 selected
paths, 10,000 visited entries and 64 folder levels. Discovered symbolic links,
Windows reparse points and special files are skipped; selecting a link as a root
is refused. Individual input files are limited to 512 MiB, in addition to engine
limits. Signed documents cannot be optimized, and encrypted PDFs requiring a
password are not supported by this command.
This is a synchronous, one-shot job. It has no folder watcher, scheduler, resume
database, concurrent file processing or GUI progress dialog. Each file has its
own temporary engine session; completed documents are not retained in memory.
+3 -3
View File
@@ -9287,9 +9287,9 @@ tier = "P1"
milestone = "M13"
status = "partial"
commands = ["actions.wizard", "ocr.recognize_batch"]
tools = ["action_run", "ocr_recognize_files"]
evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions"]
notes = "Actions and OCR run over chosen files; whole folders (with subfolders) not yet."
tools = ["action_run", "ocr_recognize_files", "doc_batch"]
evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions", "crates/automation/tests/batch.rs::batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs", "apps/pdfcraft-cli/tests/batch.rs::cli_batch_reports_partial_failure_and_continues_into_subfolders"]
notes = "Actions and OCR run over chosen files. CLI/MCP doc_batch adds bounded recursive folders for convert/ocr/optimize recipes, retained relative paths, collision-safe outputs and per-file reports. GUI folder selection, generic action recipes, scheduling and resume remain absent. See docs/cli-batch.md."
[[feature]]
id = "misc.action-prompt-steps"