mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-10-24 10:08:00 -06:00
M13: Run conversion, OCR and optimization over folder trees (#1605)
Add a shared CLI/MCP batch runner with bounded discovery, retained relative paths and per-file processing in isolated engine sessions. Publish complete outputs without overwriting existing names, and return structured errors with an optional stop-on-error policy. Cover recursive jobs, overlapping inputs, concurrent output collisions, root confinement and partial-failure reporting in automation and CLI tests. Co-authored-by: LabDennis <341377186+LabDennis@users.noreply.github.com>
This commit is contained in:
co-authored by
LabDennis
parent
73efaef13e
commit
7a0f846a28
Generated
+1
@@ -3632,6 +3632,7 @@ dependencies = [
|
||||
"png 0.18.1",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! Friendly flags for the same folder-job tool exposed by `run` and MCP.
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use crate::{CliError, OptionSpec, automation, flag, print_output, validate_options};
|
||||
|
||||
const OPTIONS: &[OptionSpec] = &[
|
||||
OptionSpec { name: "--root", takes_value: true },
|
||||
OptionSpec { name: "--out-dir", takes_value: true },
|
||||
OptionSpec { name: "--steps", takes_value: true },
|
||||
OptionSpec { name: "--recursive", takes_value: false },
|
||||
OptionSpec { name: "--stop-on-error", takes_value: false },
|
||||
OptionSpec { name: "--ppi", takes_value: true },
|
||||
OptionSpec { name: "--quality", takes_value: true },
|
||||
OptionSpec { name: "--dpi", takes_value: true },
|
||||
OptionSpec { name: "--language", takes_value: true },
|
||||
OptionSpec { name: "--ocr-all-pages", takes_value: false },
|
||||
];
|
||||
|
||||
pub(super) fn run(args: &[String]) -> Result<(), CliError> {
|
||||
validate_options("batch", args, OPTIONS)?;
|
||||
let mut paths = Vec::new();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut i = 0;
|
||||
while let Some(arg) = args.get(i) {
|
||||
if let Some(spec) = OPTIONS.iter().find(|s| s.name == arg) {
|
||||
if !seen.insert(spec.name) {
|
||||
return Err(format!("batch: {} may only be supplied once", spec.name).into());
|
||||
}
|
||||
if spec.takes_value {
|
||||
if args.get(i + 1).is_none_or(|v| v.is_empty() || v.starts_with("--")) {
|
||||
return Err(format!("batch: {} needs a value", spec.name).into());
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
} else {
|
||||
paths.push(arg);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
let steps: Vec<_> = flag(args, "--steps").ok_or("batch: missing --steps (convert, ocr, optimize)")?.split(',').collect();
|
||||
let out = flag(args, "--out-dir").ok_or("batch: missing --out-dir")?;
|
||||
let mut params =
|
||||
json!({"paths":paths,"steps":steps,"out_dir":out,"recursive":seen.contains("--recursive"),"stop_on_error":seen.contains("--stop-on-error")});
|
||||
for (option, key) in [("--ppi", "ppi"), ("--quality", "quality"), ("--dpi", "dpi")] {
|
||||
if let Some(v) = flag(args, option) {
|
||||
params[key] = serde_json::from_str(v).map_err(|_| format!("batch: {option} needs a number"))?;
|
||||
}
|
||||
}
|
||||
if let Some(lang) = flag(args, "--language") {
|
||||
params["language"] = json!(lang);
|
||||
}
|
||||
if seen.contains("--ocr-all-pages") {
|
||||
params["skip_text_pages"] = json!(false);
|
||||
}
|
||||
let mut auto = automation(args)?;
|
||||
let content = auto.call("doc_batch", ¶ms).map_err(|e| e.to_string())?;
|
||||
let failed = content.iter().any(|c| matches!(c, pdfcraft_automation::Content::Json(v) if v["failed"].as_u64().is_some_and(|n| n > 0)));
|
||||
print_output(&auto, content, None)?;
|
||||
if failed { Err("batch: one or more files failed; see the JSON report".into()) } else { Ok(()) }
|
||||
}
|
||||
@@ -17,6 +17,8 @@
|
||||
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod batch;
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode, Stdio};
|
||||
@@ -57,6 +59,9 @@ pdfcraft-cli extract <in.pdf> --pages 1,3,5 --out out.pdf
|
||||
pdfcraft-cli split <in.pdf> (--every N | --before 3,7) [--out-dir DIR]
|
||||
pdfcraft-cli check <files or dirs…> [--timeout 20] [--dpi 36] [--json out.json]
|
||||
pdfcraft-cli audit <file.pdf> [--password PW] find faux redactions (fails when found)
|
||||
pdfcraft-cli batch <files or dirs...> --steps convert,ocr,optimize --out-dir DIR
|
||||
[--recursive] [--root DIR] [--stop-on-error]
|
||||
[--ppi 150] [--quality 60] [--dpi 300] [--language en] [--ocr-all-pages]
|
||||
pdfcraft-cli tools automation tools and their JSON Schemas
|
||||
pdfcraft-cli run <tool> [key=value …] [--root DIR] [--out image.png]
|
||||
pdfcraft-cli run --script steps.json [--root DIR] [{\"tool\": \"doc_open\", \"args\": {…}}, …]
|
||||
@@ -90,6 +95,7 @@ fn main() -> ExitCode {
|
||||
Some("audit") => audit(&args[1..]),
|
||||
Some("tools") => tools(),
|
||||
Some("run") => run(&args[1..]),
|
||||
Some("batch") => batch::run(&args[1..]),
|
||||
Some("ui") => ui(&args[1..]),
|
||||
#[cfg(feature = "mcp")]
|
||||
Some("mcp") => mcp(&args[1..]),
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
use serde_json::Value;
|
||||
use std::process::Command;
|
||||
|
||||
const BIN: &str = env!("CARGO_BIN_EXE_pdfcraft-cli");
|
||||
|
||||
#[test]
|
||||
fn cli_batch_reports_partial_failure_and_continues_into_subfolders() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfcraft-cli-batch-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(dir.join("input/sub")).unwrap();
|
||||
std::fs::write(dir.join("input/0.pdf"), b"broken").unwrap();
|
||||
std::fs::write(dir.join("input/sub/page.txt"), "Batch text").unwrap();
|
||||
let output = Command::new(BIN)
|
||||
.args(["batch", "input", "--root"])
|
||||
.arg(&dir)
|
||||
.args(["--steps", "convert,optimize", "--out-dir", "out", "--recursive"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(!output.status.success());
|
||||
let report: Value = serde_json::from_slice(&output.stdout).unwrap();
|
||||
assert_eq!(report["completed"], 1, "{report}");
|
||||
assert_eq!(report["failed"], 1);
|
||||
let converted = dir.join("out/sub/page.pdf");
|
||||
assert!(converted.exists());
|
||||
let text = Command::new(BIN).arg("text").arg(converted).output().unwrap();
|
||||
assert!(text.status.success());
|
||||
assert!(String::from_utf8_lossy(&text.stdout).contains("Batch text"));
|
||||
let success = Command::new(BIN).current_dir(&dir).args(["batch", "input/sub", "--steps", "convert", "--out-dir", "other"]).output().unwrap();
|
||||
assert!(success.status.success(), "{}", String::from_utf8_lossy(&success.stderr));
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_batch_rejects_bad_flags_before_creating_outputs() {
|
||||
for args in [
|
||||
vec!["batch", "--steps"],
|
||||
vec!["batch", "--recursive", "--root"],
|
||||
vec!["batch", "--steps", "convert", "--out-dir"],
|
||||
vec!["batch", "--steps", "convert", "--steps", "ocr", "--out-dir", "out"],
|
||||
vec!["batch", "--unknown"],
|
||||
] {
|
||||
let output = Command::new(BIN).args(args).output().unwrap();
|
||||
assert!(!output.status.success());
|
||||
assert!(output.stdout.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ pdfcraft-organize = { workspace = true }
|
||||
pdfcraft-platform = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = "0.11"
|
||||
tempfile = "3.27"
|
||||
png = "0.18"
|
||||
base64 = { version = "0.22", optional = true }
|
||||
|
||||
|
||||
@@ -134,3 +134,9 @@ faces complete absent subset glyphs. `text_edit.ranges` accepts disjoint UTF-8 b
|
||||
undoable and survive saving/reopening. `image_hit` shares the UI's curve and hole hit testing.
|
||||
The bounds, limits and font requirements are in [existing-text editing](../../docs/editing-existing-text.md)
|
||||
and [image editing](../../docs/editing-images.md).
|
||||
|
||||
## Folder batches
|
||||
|
||||
`doc_batch` runs conversion, OCR and compression over files and optionally recursive
|
||||
folders. The CLI also exposes it as `pdfcraft-cli batch`. See the
|
||||
[batch guide](../../docs/cli-batch.md) for examples, output protection, errors and limits.
|
||||
|
||||
@@ -490,7 +490,7 @@ impl Automation {
|
||||
}
|
||||
|
||||
impl Automation {
|
||||
fn ocr_settings(&self, a: &Args) -> Result<pdfcraft_engine::ocr::OcrSettings> {
|
||||
pub(crate) fn ocr_settings(&self, a: &Args) -> Result<pdfcraft_engine::ocr::OcrSettings> {
|
||||
let mut settings = pdfcraft_engine::ocr::OcrSettings::default();
|
||||
if let Some(d) = a.opt_num("dpi")? {
|
||||
settings.dpi = d.clamp(72.0, 600.0) as f32;
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
//! One-shot folder jobs shared by CLI and MCP. Discovery finishes before any output is made.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use pdfcraft_engine::{Session, ocr, optimize};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{Args, Automation, Result, failed};
|
||||
|
||||
const MAX_ENTRIES: usize = 10_000;
|
||||
const MAX_DEPTH: usize = 64;
|
||||
const MAX_INPUT: u64 = 512 * 1024 * 1024;
|
||||
|
||||
struct Recipe<'a> {
|
||||
steps: Vec<&'a str>,
|
||||
images: optimize::ImageSettings,
|
||||
ocr: ocr::OcrSettings,
|
||||
}
|
||||
|
||||
impl<'a> Recipe<'a> {
|
||||
fn parse(auto: &Automation, args: &'a Args) -> Result<Self> {
|
||||
let steps = args.strs("steps")?;
|
||||
if steps.is_empty() || steps.len() > 3 {
|
||||
return Err(Args::wrong("steps", "one to three unique steps: convert, ocr, optimize"));
|
||||
}
|
||||
let mut seen = HashSet::new();
|
||||
for (i, step) in steps.iter().enumerate() {
|
||||
if !matches!(*step, "convert" | "ocr" | "optimize") || !seen.insert(*step) || (*step == "convert" && i != 0) {
|
||||
return Err(Args::wrong("steps", "unique convert/ocr/optimize steps, with convert first when present"));
|
||||
}
|
||||
}
|
||||
for (keys, step) in [(&["ppi", "quality"][..], "optimize"), (&["dpi", "language", "skip_text_pages"][..], "ocr")] {
|
||||
if !steps.contains(&step) && keys.iter().any(|k| args.get(k).is_some()) {
|
||||
return Err(failed(format!("{} settings need the {step} step", keys.join("/"))));
|
||||
}
|
||||
}
|
||||
let mut images = optimize::ImageSettings::REDUCE;
|
||||
if let Some(ppi) = args.opt_num("ppi")? {
|
||||
if !(9.0..=1600.0).contains(&ppi) {
|
||||
return Err(Args::wrong("ppi", "9 to 1600"));
|
||||
}
|
||||
images.target_ppi = ppi;
|
||||
images.above_ppi = ppi * 1.5;
|
||||
}
|
||||
if let Some(q) = args.opt_int("quality")? {
|
||||
if !(1..=100).contains(&q) {
|
||||
return Err(Args::wrong("quality", "1 to 100"));
|
||||
}
|
||||
images.compression = optimize::Compression::Jpeg(q as u8);
|
||||
}
|
||||
if args.opt_num("dpi")?.is_some_and(|dpi| !(72.0..=600.0).contains(&dpi)) {
|
||||
return Err(Args::wrong("dpi", "72 to 600"));
|
||||
}
|
||||
Ok(Self { steps, images, ocr: auto.ocr_settings(args)? })
|
||||
}
|
||||
|
||||
fn accepts(&self, path: &Path) -> bool {
|
||||
let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_ascii_lowercase();
|
||||
extension == "pdf" || (self.steps.contains(&"convert") && pdfcraft_engine::CONVERTIBLE.contains(&extension.as_str()))
|
||||
}
|
||||
|
||||
fn process(&self, path: &Path) -> Result<(Arc<Vec<u8>>, Vec<Value>)> {
|
||||
// A fresh session per file bounds retained document state and never touches caller tabs.
|
||||
let mut session = Session::new();
|
||||
let mut input = Vec::new();
|
||||
std::fs::File::open(path).map_err(failed)?.take(MAX_INPUT + 1).read_to_end(&mut input).map_err(failed)?;
|
||||
if input.len() as u64 > MAX_INPUT {
|
||||
return Err(failed("input exceeds 512 MiB"));
|
||||
}
|
||||
let name = path.file_name().and_then(|n| n.to_str()).ok_or_else(|| failed("input name is not UTF-8"))?;
|
||||
let mut bytes = Arc::new(input);
|
||||
let mut log = Vec::new();
|
||||
for step in &self.steps {
|
||||
let before = bytes.len();
|
||||
match *step {
|
||||
"convert" => bytes = session.convert_to_pdf(name, &bytes)?.1,
|
||||
"ocr" => {
|
||||
let engine = ocr::engine().map_err(failed)?;
|
||||
let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &engine, |_, _| true).map_err(failed)?;
|
||||
log.push(
|
||||
json!({"step":"ocr","words":result.words(),"skipped_pages":result.pages.iter().filter(|p| p.skipped.is_some()).count()}),
|
||||
);
|
||||
bytes = result.bytes;
|
||||
continue;
|
||||
}
|
||||
"optimize" => {
|
||||
let id = session.open(name, None, bytes.clone(), None).map_err(failed)?;
|
||||
let settings = optimize::Settings { color: self.images, gray: self.images, ..Default::default() };
|
||||
let (candidate, _) = session.optimized_bytes(id, &settings, &[])?;
|
||||
session.close(id);
|
||||
if candidate.len() < bytes.len() {
|
||||
bytes = candidate;
|
||||
}
|
||||
}
|
||||
_ => return Err(failed("unknown batch step")),
|
||||
}
|
||||
log.push(json!({"step":step,"bytes_before":before,"bytes_after":bytes.len()}));
|
||||
}
|
||||
Ok((bytes, log))
|
||||
}
|
||||
}
|
||||
|
||||
// Windows junctions are reparse points too; do not descend into them as ordinary folders.
|
||||
fn linked(metadata: &std::fs::Metadata) -> bool {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
metadata.file_attributes() & 0x400 != 0
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
metadata.file_type().is_symlink()
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonicalize the existing ancestor, including when the requested output does not exist yet.
|
||||
fn absolute(path: &Path) -> Result<PathBuf> {
|
||||
let mut ancestor = path;
|
||||
let mut tail = Vec::new();
|
||||
while !ancestor.exists() {
|
||||
if ancestor.symlink_metadata().is_ok() {
|
||||
return Err(failed("broken link in batch path"));
|
||||
}
|
||||
tail.push(ancestor.file_name().ok_or_else(|| failed("invalid batch path"))?);
|
||||
ancestor = ancestor.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
|
||||
}
|
||||
let mut path = ancestor.canonicalize().map_err(failed)?;
|
||||
path.extend(tail.iter().rev());
|
||||
crate::lexical(&path).ok_or_else(|| failed("invalid batch path"))
|
||||
}
|
||||
|
||||
struct Discovery {
|
||||
files: BTreeMap<PathBuf, PathBuf>,
|
||||
skipped: Vec<Value>,
|
||||
seen: HashSet<PathBuf>,
|
||||
visited: usize,
|
||||
}
|
||||
|
||||
impl Discovery {
|
||||
fn walk(&mut self, path: &Path, relative: &Path, out: &Path, recursive: bool, depth: usize) -> Result<()> {
|
||||
self.visited += 1;
|
||||
if self.visited > MAX_ENTRIES || depth > MAX_DEPTH {
|
||||
return Err(failed("batch discovery exceeds 10000 entries or 64 folder levels"));
|
||||
}
|
||||
if depth > 0 && path.starts_with(out) {
|
||||
return Ok(());
|
||||
}
|
||||
let metadata = path.symlink_metadata().map_err(|e| failed(format!("{}: {e}", path.display())))?;
|
||||
if linked(&metadata) || (!metadata.is_file() && !metadata.is_dir()) {
|
||||
self.skipped.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":"link or special file"}));
|
||||
} else if (metadata.is_file() || depth == 0 || recursive) && self.seen.insert(path.to_path_buf()) {
|
||||
if metadata.is_file() {
|
||||
self.files.insert(path.to_path_buf(), relative.to_path_buf());
|
||||
} else if depth == 0 || recursive {
|
||||
// Bound the list before sorting it; never collect an unbounded folder listing.
|
||||
let mut entries = Vec::new();
|
||||
for entry in std::fs::read_dir(path).map_err(failed)? {
|
||||
if entries.len() >= MAX_ENTRIES.saturating_sub(self.visited) {
|
||||
return Err(failed("batch discovery exceeds 10000 entries"));
|
||||
}
|
||||
entries.push(entry.map_err(failed)?.path());
|
||||
}
|
||||
entries.sort();
|
||||
for child in entries {
|
||||
let name = child.file_name().ok_or_else(|| failed("entry has no name"))?;
|
||||
self.walk(&child, &relative.join(name), out, recursive, depth + 1)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Automation {
|
||||
pub(crate) fn doc_batch(&self, args: &Args) -> Result<Value> {
|
||||
let recipe = Recipe::parse(self, args)?;
|
||||
let recursive = args.opt_bool("recursive")?.unwrap_or(false);
|
||||
let stop = args.opt_bool("stop_on_error")?.unwrap_or(false);
|
||||
let paths = args.strs("paths")?;
|
||||
if paths.is_empty() || paths.len() > 1000 {
|
||||
return Err(Args::wrong("paths", "1 to 1000 files or folders"));
|
||||
}
|
||||
let out = absolute(&self.resolve(args.str("out_dir")?, true)?)?;
|
||||
let mut roots = Vec::new();
|
||||
for path in paths {
|
||||
let resolved = self.resolve(path, false)?;
|
||||
let original = self.root.as_ref().map_or_else(|| PathBuf::from(path), |root| root.join(path));
|
||||
if linked(&original.symlink_metadata().map_err(failed)?) {
|
||||
return Err(failed("input roots must be regular files or folders, not links"));
|
||||
}
|
||||
let real = absolute(&resolved)?;
|
||||
if real.is_dir() && real.starts_with(&out) {
|
||||
return Err(failed("out_dir must not contain an input folder or equal it"));
|
||||
}
|
||||
roots.push(real);
|
||||
}
|
||||
let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: 0 };
|
||||
for root in roots {
|
||||
let relative = if root.is_dir() { PathBuf::new() } else { PathBuf::from(root.file_name().ok_or_else(|| failed("input has no name"))?) };
|
||||
discovery.walk(&root, &relative, &out, recursive, 0)?;
|
||||
}
|
||||
let mut files = discovery.skipped;
|
||||
let (mut completed, mut failed_count) = (0, 0);
|
||||
let mut stopped = false;
|
||||
for (path, relative) in discovery.files {
|
||||
if stopped || !recipe.accepts(&path) {
|
||||
files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}}));
|
||||
continue;
|
||||
}
|
||||
// Recheck confinement and links before reading a discovered file.
|
||||
let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?;
|
||||
if linked(&path.symlink_metadata().map_err(failed)?) || absolute(¤t)? != path {
|
||||
return Err(failed("input path changed after discovery"));
|
||||
}
|
||||
let (bytes, log) = recipe.process(¤t)?;
|
||||
let target = out.join(relative).with_extension("pdf");
|
||||
let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?;
|
||||
if !target.starts_with(&out) {
|
||||
return Err(failed("output path leaves out_dir"));
|
||||
}
|
||||
let output = publish(&target, &bytes)?;
|
||||
Ok((output, bytes.len(), log))
|
||||
}))
|
||||
.unwrap_or_else(|_| Err(failed("batch conversion panicked; no output was published")));
|
||||
match result {
|
||||
Ok((output, bytes, steps)) => {
|
||||
completed += 1;
|
||||
files.push(json!({"path":path.to_string_lossy(),"status":"ok","output":output.to_string_lossy(),"bytes":bytes,"steps":steps}));
|
||||
}
|
||||
Err(e) => {
|
||||
failed_count += 1;
|
||||
files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()}));
|
||||
stopped = stop;
|
||||
}
|
||||
}
|
||||
}
|
||||
let skipped = files.len().saturating_sub(completed + failed_count);
|
||||
Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files}))
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish atomically without replacing any existing name, even if another writer wins a race.
|
||||
fn publish(target: &Path, bytes: &[u8]) -> Result<PathBuf> {
|
||||
let dir = target.parent().ok_or_else(|| failed("output has no parent"))?;
|
||||
std::fs::create_dir_all(dir).map_err(failed)?;
|
||||
let mut temp = tempfile::NamedTempFile::new_in(dir).map_err(failed)?;
|
||||
temp.write_all(bytes).map_err(failed)?;
|
||||
temp.as_file().sync_all().map_err(failed)?;
|
||||
let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy();
|
||||
for n in 1..=10_000 {
|
||||
let candidate = if n == 1 { target.to_path_buf() } else { dir.join(format!("{stem} ({n}).pdf")) };
|
||||
match temp.persist_noclobber(&candidate) {
|
||||
Ok(_) => return Ok(candidate),
|
||||
Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists || candidate.symlink_metadata().is_ok() => temp = e.file,
|
||||
Err(e) => return Err(failed(e.error)),
|
||||
}
|
||||
}
|
||||
Err(failed("all 10000 output names are occupied"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn discovery_limits_fail_before_reading_another_entry() {
|
||||
let mut discovery = Discovery { files: BTreeMap::new(), skipped: Vec::new(), seen: HashSet::new(), visited: MAX_ENTRIES };
|
||||
assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, 0).unwrap_err().to_string().contains("10000"));
|
||||
discovery.visited = 0;
|
||||
assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64"));
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod a11y;
|
||||
mod batch;
|
||||
mod comments;
|
||||
mod content;
|
||||
mod conventions;
|
||||
@@ -165,6 +166,7 @@ impl Automation {
|
||||
let a = Args(args);
|
||||
let out = match name {
|
||||
"doc_open" => self.doc_open(&a)?,
|
||||
"doc_batch" => self.doc_batch(&a)?,
|
||||
"doc_list" => json!({ "documents": self.session.docs().iter().map(summary).collect::<Vec<_>>() }),
|
||||
"doc_info" => {
|
||||
let id = self.doc(&a)?.id;
|
||||
|
||||
@@ -329,7 +329,7 @@ fn guarded_tool(f: impl FnOnce() -> Result<Value, (i64, String)>) -> Result<Valu
|
||||
}
|
||||
|
||||
fn tool_content(name: &str, content: Vec<Content>) -> Value {
|
||||
let failed = name == "command_batch"
|
||||
let failed = matches!(name, "command_batch" | "doc_batch")
|
||||
&& content.iter().any(|c| match c {
|
||||
Content::Json(v) => v.get("failed").and_then(Value::as_u64).is_some_and(|n| n > 0),
|
||||
_ => false,
|
||||
|
||||
@@ -118,6 +118,17 @@ pub fn tools() -> Vec<ToolDef> {
|
||||
let save_out = json!({ "type": "string", "description": "File to write. Omit to open the result as a new unsaved document instead." });
|
||||
let open = json!({ "type": "boolean", "description": "Also open the result as a new document (default: only when out is omitted)." });
|
||||
vec![
|
||||
t("doc_batch", "Process files and folders", "Run convert, ocr and/or optimize over files and folders, one file at a time. recursive defaults to false. Preserve paths relative to each selected folder inside out_dir; duplicate inputs run once. Skip unsupported files, links and the output subtree. Existing outputs receive numbered names and are never replaced. Continue after per-file errors unless stop_on_error is true. Return completed/failed/skipped counts and per-file results. Discovery is bounded to 10000 entries and 64 folder levels. Convert must be first; each step can appear once. OCR uses installed models. Optimize uses Reduce File Size settings, optionally overriding target ppi (threshold 1.5 times ppi) and JPEG quality; a larger candidate keeps the original bytes. Sources and open documents are unchanged.")
|
||||
.with(schema(json!({
|
||||
"paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000},
|
||||
"out_dir": {"type":"string"},
|
||||
"steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true},
|
||||
"recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"},
|
||||
"ppi": {"type":"number","minimum":9,"maximum":1600},
|
||||
"quality": {"type":"integer","minimum":1,"maximum":100},
|
||||
"dpi": {"type":"number","minimum":72,"maximum":600},
|
||||
"language": {"type":"string"}, "skip_text_pages": {"type":"boolean"}
|
||||
}), &["paths","out_dir","steps"])),
|
||||
t("doc_open", "Open a PDF", "Open a PDF file and return its document id, page count and whether it can be edited.")
|
||||
.cmd("file.open")
|
||||
.with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])),
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
use pdfcraft_automation::{Automation, Content};
|
||||
use pdfcraft_engine::Session;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn result(auto: &mut Automation, args: Value) -> Value {
|
||||
match auto.call("doc_batch", &args).unwrap().remove(0) {
|
||||
Content::Json(value) => value,
|
||||
_ => panic!("expected a report"),
|
||||
}
|
||||
}
|
||||
|
||||
fn pdf() -> Vec<u8> {
|
||||
Session::new().create_from_text("Original", "Original text").unwrap().as_ref().clone()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("input/output/sub")).unwrap();
|
||||
std::fs::write(dir.path().join("input/a.txt"), "Hello").unwrap();
|
||||
std::fs::write(dir.path().join("input/b.pdf"), b"broken").unwrap();
|
||||
std::fs::write(dir.path().join("input/sub/a.txt"), "World").unwrap();
|
||||
std::fs::write(dir.path().join("input/notes.bin"), b"ignore").unwrap();
|
||||
std::fs::write(dir.path().join("input/output/sub/a.pdf"), b"precious").unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let args = json!({"paths":["input","input/a.txt"],"steps":["convert","optimize"],"out_dir":"input/output","recursive":true});
|
||||
let report = result(&mut auto, args.clone());
|
||||
assert_eq!(report["completed"], 2, "{report}");
|
||||
assert_eq!(report["failed"], 1);
|
||||
assert_eq!(report["skipped"], 1);
|
||||
assert_eq!(std::fs::read(dir.path().join("input/output/sub/a.pdf")).unwrap(), b"precious");
|
||||
assert_eq!(std::fs::read_to_string(dir.path().join("input/a.txt")).unwrap(), "Hello");
|
||||
for name in ["a.pdf", "sub/a (2).pdf"] {
|
||||
let bytes = std::fs::read(dir.path().join("input/output").join(name)).unwrap();
|
||||
let mut session = Session::new();
|
||||
let id = session.open(name, None, std::sync::Arc::new(bytes), None).unwrap();
|
||||
assert_eq!(session.get(id).unwrap().info.pages.len(), 1);
|
||||
}
|
||||
assert!(!dir.path().join("input/output/b.pdf").exists());
|
||||
assert!(auto.session().docs().is_empty());
|
||||
let again = result(&mut auto, args);
|
||||
assert_eq!(again["completed"], 2, "previous outputs must not become new inputs: {again}");
|
||||
assert!(dir.path().join("input/output/a (2).pdf").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_is_nonrecursive_by_default_and_stop_reports_unprocessed_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
|
||||
std::fs::write(dir.path().join("input/0.pdf"), b"broken").unwrap();
|
||||
std::fs::write(dir.path().join("input/1.pdf"), pdf()).unwrap();
|
||||
std::fs::write(dir.path().join("input/sub/2.pdf"), pdf()).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","stop_on_error":true}));
|
||||
assert_eq!(report["failed"], 1);
|
||||
assert_eq!(report["completed"], 0);
|
||||
assert_eq!(report["skipped"], 1);
|
||||
assert_eq!(report["stopped"], true);
|
||||
assert!(!dir.path().join("out").exists());
|
||||
let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"}));
|
||||
assert_eq!(report["completed"], 1);
|
||||
assert!(!dir.path().join("out/sub").exists());
|
||||
assert!(std::fs::metadata(dir.path().join("out/1.pdf")).unwrap().len() <= pdf().len() as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_batch_arguments_and_outside_paths_write_nothing() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(dir.path().join("input")).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
for change in [
|
||||
json!({"steps":["convert","convert"]}),
|
||||
json!({"steps":["optimize","convert"]}),
|
||||
json!({"steps":[]}),
|
||||
json!({"steps":["nope"]}),
|
||||
json!({"recursive":"yes"}),
|
||||
json!({"paths":[]}),
|
||||
json!({"paths":["input","../outside"]}),
|
||||
json!({"out_dir":"../outside"}),
|
||||
json!({"out_dir":"input"}),
|
||||
json!({"out_dir":"."}),
|
||||
json!({"quality":0}),
|
||||
json!({"quality":"80"}),
|
||||
json!({"ppi":2000}),
|
||||
json!({"steps":["ocr"],"language":"not-a-language"}),
|
||||
json!({"steps":["ocr"],"dpi":1}),
|
||||
] {
|
||||
let mut args = json!({"paths":["input"],"steps":["optimize"],"out_dir":"out"});
|
||||
args.as_object_mut().unwrap().extend(change.as_object().unwrap().clone());
|
||||
assert!(auto.call("doc_batch", &args).is_err(), "{args}");
|
||||
assert!(!dir.path().join("out").exists(), "invalid arguments must be refused before writing");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicitly_selected_subfolder_is_processed_without_recursive() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
|
||||
std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["convert"],"out_dir":"out"}));
|
||||
assert_eq!(report["completed"], 1, "{report}");
|
||||
assert!(dir.path().join("out/page.pdf").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_file_can_write_a_numbered_copy_beside_its_source() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"."}));
|
||||
assert_eq!(report["completed"], 1, "{report}");
|
||||
assert_eq!(std::fs::read(dir.path().join("page.pdf")).unwrap(), pdf());
|
||||
assert_eq!(std::fs::read(dir.path().join("page (2).pdf")).unwrap(), pdf());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn non_utf8_discovered_names_are_reported_without_panicking() {
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(dir.path().join("input")).unwrap();
|
||||
std::fs::write(dir.path().join("input").join(std::ffi::OsStr::from_bytes(b"bad-\xff.pdf")), pdf()).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out"}));
|
||||
assert_eq!(report["failed"], 1);
|
||||
assert!(report["files"][0]["error"].as_str().unwrap().contains("UTF-8"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn directory_links_are_not_followed_and_output_links_cannot_escape() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("input/sub")).unwrap();
|
||||
std::fs::create_dir(dir.path().join("outside")).unwrap();
|
||||
std::fs::create_dir(dir.path().join("out")).unwrap();
|
||||
std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap();
|
||||
std::os::unix::fs::symlink(dir.path().join("input"), dir.path().join("input/loop")).unwrap();
|
||||
std::os::unix::fs::symlink(dir.path().join("outside"), dir.path().join("out/sub")).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["input"],"steps":["convert"],"out_dir":"out","recursive":true}));
|
||||
assert_eq!(report["skipped"], 1);
|
||||
assert_eq!(report["failed"], 1);
|
||||
assert_eq!(std::fs::read_dir(dir.path().join("outside")).unwrap().count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_ocr_uses_existing_models_or_reports_the_missing_models_per_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
let report = result(&mut auto, json!({"paths":["page.pdf"],"steps":["ocr","optimize"],"out_dir":"out"}));
|
||||
if pdfcraft_engine::ocr::available() {
|
||||
assert_eq!(report["completed"], 1, "{report}");
|
||||
assert_eq!(report["files"][0]["steps"][0]["skipped_pages"], 1);
|
||||
assert!(dir.path().join("out/page.pdf").exists());
|
||||
} else {
|
||||
assert_eq!(report["failed"], 1, "{report}");
|
||||
assert!(report["files"][0]["error"].as_str().unwrap().contains("model"), "{report}");
|
||||
assert!(!dir.path().join("out/page.pdf").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_output_collisions_are_safe_under_concurrent_writers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("page.pdf"), pdf()).unwrap();
|
||||
let barrier = std::sync::Barrier::new(2);
|
||||
std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = (0..2)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
let mut auto = Automation::new().with_root(dir.path()).unwrap();
|
||||
barrier.wait();
|
||||
result(&mut auto, json!({"paths":["page.pdf"],"steps":["convert"],"out_dir":"out"}))
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let reports: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
|
||||
assert!(reports.iter().all(|r| r["completed"] == 1), "{reports:?}");
|
||||
assert_ne!(reports[0]["files"][0]["output"], reports[1]["files"][0]["output"]);
|
||||
});
|
||||
for name in ["page.pdf", "page (2).pdf"] {
|
||||
assert_eq!(std::fs::read(dir.path().join("out").join(name)).unwrap(), pdf());
|
||||
}
|
||||
assert_eq!(std::fs::read_dir(dir.path().join("out")).unwrap().count(), 2, "no staging files remain");
|
||||
}
|
||||
|
||||
#[cfg(feature = "mcp")]
|
||||
#[test]
|
||||
fn mcp_batch_errors_keep_the_structured_per_file_report() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("broken.pdf"), b"broken").unwrap();
|
||||
let input = json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"doc_batch","arguments":{"paths":["broken.pdf"],"steps":["optimize"],"out_dir":"out"}}}).to_string() + "\n";
|
||||
let mut out = Vec::new();
|
||||
pdfcraft_automation::mcp::McpServer::new(Automation::new().with_root(dir.path()).unwrap()).serve(input.as_bytes(), &mut out).unwrap();
|
||||
let report: Value = serde_json::from_slice(&out).unwrap();
|
||||
assert_eq!(report["result"]["isError"], true, "{report}");
|
||||
assert_eq!(report["result"]["structuredContent"]["failed"], 1);
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
# Batch conversion, OCR and compression
|
||||
|
||||
`pdfcraft-cli batch` runs the same steps over files or folders, one file at a time.
|
||||
It writes separate PDF outputs and a JSON report to stdout. It does not change
|
||||
source files or documents open in the editor.
|
||||
|
||||
```sh
|
||||
# Compress all PDFs, including subfolders.
|
||||
pdfcraft-cli batch scans --root . --recursive --steps optimize --out-dir compressed
|
||||
|
||||
# Convert images and text, then compress. Existing PDFs are accepted too.
|
||||
pdfcraft-cli batch inputs --root . --recursive --steps convert,optimize --out-dir pdfs --ppi 150 --quality 80
|
||||
|
||||
# Add searchable text before compression. Requires the installed OCR models.
|
||||
pdfcraft-cli batch scans --root . --recursive --steps ocr,optimize --out-dir searchable
|
||||
```
|
||||
|
||||
Multiple input files or folders can be listed before the options. Directories are
|
||||
read only one level deep unless `--recursive` is present. Paths within each input
|
||||
folder are retained inside the output folder. Files encountered more than once
|
||||
through overlapping selections run once; the first selection determines their
|
||||
relative output path. Processing order is sorted by input path.
|
||||
|
||||
Steps are `convert`, `ocr` and `optimize`, each at most once. `convert`, when used,
|
||||
must come first. Conversion uses the same supported extensions as Create PDF;
|
||||
new engine importers become available here automatically. Without `convert`,
|
||||
only `.pdf` inputs are processed. Unsupported extensions are reported as skipped.
|
||||
|
||||
- `optimize` uses Reduce File Size settings: JPEG quality 60, resampling to 150 ppi
|
||||
above 225 ppi, plus the existing document cleanup. `--ppi` (9–1600) sets the target
|
||||
and a threshold of 1.5 times that value; `--quality` (1–100) sets JPEG quality.
|
||||
These settings can lose image detail. A candidate that is not smaller retains
|
||||
the bytes entering that step. This is not an images-only or lossless preset.
|
||||
- `ocr` uses the installed engine and its supported languages. `--dpi` (72–600,
|
||||
default 300), `--language` (default `en`) and `--ocr-all-pages` configure it.
|
||||
By default pages already containing text are skipped. Missing models are
|
||||
reported as file errors; nothing is downloaded or installed by the batch.
|
||||
- `--stop-on-error` stops processing after the first file error. By default the
|
||||
remaining files continue. Earlier successful outputs stay, and unprocessed
|
||||
files are listed as skipped. A file is published only after all its steps succeed.
|
||||
|
||||
Existing output names are never replaced, even by concurrent batch writers:
|
||||
`book.pdf`, `book (2).pdf`, and so on. A temporary file in the destination folder
|
||||
is published without overwriting. Sources are never moved or deleted. An output
|
||||
folder inside an input folder is excluded from discovery, including on reruns.
|
||||
The output folder cannot equal or contain a selected input folder.
|
||||
|
||||
The report contains `completed`, `failed`, `skipped`, `stopped` and `files`.
|
||||
Each file records its status, output and step results, or the error/skip reason.
|
||||
The `batch` command exits nonzero on any file failure while still printing the
|
||||
report. Redirect stdout to retain it; keep the report outside the input folders.
|
||||
|
||||
## MCP and JSON scripts
|
||||
|
||||
The same implementation is available as `doc_batch`:
|
||||
|
||||
```json
|
||||
{
|
||||
"paths": ["inputs"],
|
||||
"steps": ["convert", "optimize"],
|
||||
"recursive": true,
|
||||
"out_dir": "pdfs",
|
||||
"ppi": 150,
|
||||
"quality": 80,
|
||||
"stop_on_error": false
|
||||
}
|
||||
```
|
||||
|
||||
In `pdfcraft-cli run --script`, put this object in the `args` of a `doc_batch`
|
||||
step. MCP marks a report with failed files as `isError: true` and retains the
|
||||
structured report. Tool names and OCR settings match the existing automation API
|
||||
(`skip_text_pages: false` corresponds to `--ocr-all-pages`).
|
||||
|
||||
## Boundaries
|
||||
|
||||
Discovery completes before writing. Invalid arguments, unreadable input folders
|
||||
or paths outside `--root` refuse the call. Traversal accepts at most 1,000 selected
|
||||
paths, 10,000 visited entries and 64 folder levels. Discovered symbolic links,
|
||||
Windows reparse points and special files are skipped; selecting a link as a root
|
||||
is refused. Individual input files are limited to 512 MiB, in addition to engine
|
||||
limits. Signed documents cannot be optimized, and encrypted PDFs requiring a
|
||||
password are not supported by this command.
|
||||
|
||||
This is a synchronous, one-shot job. It has no folder watcher, scheduler, resume
|
||||
database, concurrent file processing or GUI progress dialog. Each file has its
|
||||
own temporary engine session; completed documents are not retained in memory.
|
||||
@@ -9287,9 +9287,9 @@ tier = "P1"
|
||||
milestone = "M13"
|
||||
status = "partial"
|
||||
commands = ["actions.wizard", "ocr.recognize_batch"]
|
||||
tools = ["action_run", "ocr_recognize_files"]
|
||||
evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions"]
|
||||
notes = "Actions and OCR run over chosen files; whole folders (with subfolders) not yet."
|
||||
tools = ["action_run", "ocr_recognize_files", "doc_batch"]
|
||||
evidence = ["crates/automation/tests/automation.rs::actions_through_tools", "crates/ui-egui/tests/actions.rs::run_and_create_actions", "crates/automation/tests/batch.rs::batch_recurses_preserves_paths_continues_errors_and_never_replaces_outputs", "apps/pdfcraft-cli/tests/batch.rs::cli_batch_reports_partial_failure_and_continues_into_subfolders"]
|
||||
notes = "Actions and OCR run over chosen files. CLI/MCP doc_batch adds bounded recursive folders for convert/ocr/optimize recipes, retained relative paths, collision-safe outputs and per-file reports. GUI folder selection, generic action recipes, scheduling and resume remain absent. See docs/cli-batch.md."
|
||||
|
||||
[[feature]]
|
||||
id = "misc.action-prompt-steps"
|
||||
|
||||
Reference in New Issue
Block a user