Compare commits

..
8 changed files with 77 additions and 90 deletions
+2 -2
View File
File diff suppressed because one or more lines are too long
+1 -17
View File
@@ -132,23 +132,7 @@ impl Server {
if batch.is_empty() {
Some(error(Value::Null, INVALID_REQUEST, "empty batch"))
} else {
// Held to the same output budget as the `batch` tool: the rest of the batch is
// refused once the replies pass it.
let mut replies: Vec<Value> = Vec::new();
let mut size = 0usize;
for m in batch {
if size > crate::tools::MAX_BATCH_OUTPUT {
// Notifications get no reply, run or not.
if let Some(id) = m.get("id").cloned() {
replies.push(error(id, INTERNAL_ERROR, "the batch's replies passed the output limit; not run"));
}
continue;
}
if let Some(r) = self.handle(m) {
size = size.saturating_add(crate::tools::json_len(&r));
replies.push(r);
}
}
let replies: Vec<Value> = batch.into_iter().filter_map(|m| self.handle(m)).collect();
(!replies.is_empty()).then_some(Value::Array(replies))
}
}
+4 -68
View File
@@ -202,26 +202,6 @@ fn with_active_id(b: &mut dyn Backend, r: Result<Value, String>) -> Result<Value
Ok(v)
}
/// Most bytes of results one `batch` (or one legacy JSON-RPC batch) keeps for its reply.
pub const MAX_BATCH_OUTPUT: usize = 64 << 20;
/// Length of `v` written as JSON, counted without keeping the text.
pub(crate) fn json_len(v: &Value) -> usize {
struct Count(usize);
impl std::io::Write for Count {
fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
self.0 = self.0.saturating_add(b.len());
Ok(b.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut n = Count(0);
let _ = serde_json::to_writer(&mut n, v);
n.0
}
fn wrap(r: Result<Value, String>) -> ToolResult {
match r {
Ok(v) => ToolResult::json(&v),
@@ -261,25 +241,15 @@ pub fn call_tool(b: &mut dyn Backend, name: &str, a: &Value) -> ToolResult {
"batch" => {
let keep = a.get("keepGoing").and_then(Value::as_bool).unwrap_or(false);
let mut out = Vec::new();
// What the results hold so far: past MAX_BATCH_OUTPUT the batch stops, rather than
// keeping copies of a document command after command until the reply is written.
let mut size = 0usize;
for c in a.get("commands").and_then(Value::as_array).cloned().unwrap_or_default() {
let id = c.get("command").and_then(Value::as_str).unwrap_or("");
let r = exec(b, id, c.get("params").cloned().unwrap_or(json!({})), doc);
let mut failed = r.is_err();
let entry = match r {
let failed = r.is_err();
out.push(match r {
Ok(v) => json!({"command": id, "ok": true, "result": v}),
Err(e) => json!({"command": id, "ok": false, "error": e}),
};
size = size.saturating_add(json_len(&entry));
if size > MAX_BATCH_OUTPUT {
out.push(json!({"command": id, "ok": false, "error": format!("the batch's results passed {MAX_BATCH_OUTPUT} bytes; this command's result and the commands after it are left out")}));
failed = true;
} else {
out.push(entry);
}
if failed && (!keep || size > MAX_BATCH_OUTPUT) {
});
if failed && !keep {
return ToolResult { content: vec![json!({"type": "text", "text": Value::Array(out).to_string()})], is_error: true };
}
}
@@ -414,40 +384,6 @@ mod tests {
assert!(b.seen.take().expect("called").1.get("doc").is_none());
}
/// A backend whose every result is 1 MiB of text (a whole document, say).
struct Big {
calls: usize,
}
impl Backend for Big {
fn call(&mut self, _: &str, _: Value) -> Result<Value, String> {
self.calls += 1;
Ok(json!("x".repeat(1 << 20)))
}
fn has_ui(&self) -> bool {
false
}
fn describe(&self) -> String {
"big".into()
}
}
#[test]
fn a_batch_stops_once_its_results_pass_the_output_budget() {
// 200 commands of 1 MiB results each kept 200 MiB until the reply; it stops at 64 MiB.
let commands: Vec<Value> = (0..200).map(|_| json!({"command": "edit.copy"})).collect();
let mut b = Big { calls: 0 };
let r = call_tool(&mut b, "batch", &json!({"commands": commands, "keepGoing": true}));
assert!(r.is_error);
assert!(b.calls <= (MAX_BATCH_OUTPUT >> 20) + 1, "{} commands ran", b.calls);
let text = r.content[0]["text"].as_str().unwrap_or_default();
assert!(text.len() <= MAX_BATCH_OUTPUT + 1024, "{}", text.len());
assert!(text.contains("left out"));
// A small batch is untouched.
let r = call_tool(&mut b, "batch", &json!({"commands": [{"command": "edit.copy"}]}));
assert!(!r.is_error);
}
/// The password has to be in the schema, or a client has no way to know it can be sent.
#[test]
fn the_schema_advertises_the_password_as_optional() {
+1
View File
@@ -77,6 +77,7 @@ fn write_attempt(doc: &Document, lay: &DocLayout, opts: &PdfOptions, pages: &[us
cmaps: HashMap::new(),
images: HashMap::new(),
vectors: HashMap::new(),
vector_bytes_left: crate::exporter::MAX_CACHED_VECTOR_BYTES,
tags: Vec::new(),
tag_index: HashMap::new(),
links: Vec::new(),
+16 -2
View File
@@ -15,6 +15,9 @@ mod draw;
mod nav;
mod text;
/// Most bytes of parsed metafiles (geometry, text and pixels) one export keeps between pages.
pub(crate) const MAX_CACHED_VECTOR_BYTES: usize = 256 << 20;
pub(crate) struct Exporter<'a> {
pub(crate) doc: &'a Document,
pub(crate) lay: &'a DocLayout,
@@ -30,6 +33,8 @@ pub(crate) struct Exporter<'a> {
pub(crate) images: HashMap<String, Option<Image>>,
/// Parsed metafiles by media key (`None`: not readable).
pub(crate) vectors: HashMap<String, Option<Arc<Vector>>>,
/// Bytes of parsed metafiles `vectors` may still keep ([`MAX_CACHED_VECTOR_BYTES`]).
pub(crate) vector_bytes_left: usize,
pub(crate) tags: Vec<TagEntry>,
pub(crate) tag_index: HashMap<(StoryRef, DocPath), usize>,
/// Link rectangles on the current page: (x0, y0, x1, y1, target).
@@ -84,7 +89,7 @@ impl Exporter<'_> {
/// The metafile picture for a media key, parsed once. None when the bytes are not a metafile
/// (the raster path takes them) or do not parse (the raster path then draws the grey box).
fn vector(&mut self, key: &str) -> Option<Arc<Vector>> {
pub(crate) fn vector(&mut self, key: &str) -> Option<Arc<Vector>> {
let doc = self.doc;
let bytes = doc.media.get(key)?;
if !wordcraft_metafile::is_metafile(bytes) {
@@ -97,7 +102,16 @@ impl Exporter<'_> {
if v.is_none() {
log::warn!("PDF: metafile `{key}` could not be read");
}
self.vectors.insert(key.to_string(), v.clone());
// Kept for the pages that draw it again while the cache has room; past that it is parsed
// again when it is next drawn, rather than every metafile's geometry piling up.
let size = v.as_ref().map_or(0, |v| v.heap_bytes());
match self.vector_bytes_left.checked_sub(size) {
Some(left) => {
self.vector_bytes_left = left;
self.vectors.insert(key.to_string(), v.clone());
}
None => log::debug!("PDF: metafile `{key}` ({size} bytes parsed) is not kept between pages"),
}
v
}
+17
View File
@@ -14,6 +14,23 @@ pub(crate) struct Vector {
images: Vec<Option<Image>>,
}
impl Vector {
/// Bytes the parsed picture holds: its paths' segments, its text and its bitmaps' pixels.
pub(crate) fn heap_bytes(&self) -> usize {
let item = |it: &wordcraft_metafile::Item| match it {
wordcraft_metafile::Item::Path { segs, .. } => segs.len().saturating_mul(std::mem::size_of::<wordcraft_metafile::Seg>()),
wordcraft_metafile::Item::Bitmap { width, height, .. } => (*width as usize).saturating_mul(*height as usize).saturating_mul(4),
wordcraft_metafile::Item::Text(t) => t.text.len().saturating_add(t.font.len()),
};
let items = self
.pic
.items
.iter()
.fold(self.pic.items.len().saturating_mul(std::mem::size_of::<wordcraft_metafile::Item>()), |n, it| n.saturating_add(item(it)));
items.saturating_add(self.images.len().saturating_mul(std::mem::size_of::<Option<Image>>()))
}
}
/// Parses a metafile and embeds its bitmaps. None when the picture has no usable size.
pub(crate) fn build_vector(bytes: &[u8]) -> Option<Vector> {
let mut pic = wordcraft_metafile::parse(bytes).ok()?;
+35
View File
@@ -1367,6 +1367,7 @@ fn bookmark_destination_is_the_page_of_its_start() {
cmaps: HashMap::new(),
images: HashMap::new(),
vectors: HashMap::new(),
vector_bytes_left: crate::exporter::MAX_CACHED_VECTOR_BYTES,
tags: Vec::new(),
tag_index: HashMap::new(),
links: Vec::new(),
@@ -2488,3 +2489,37 @@ fn baseline_is_the_most_shared_y() {
assert_eq!(baseline(&[(1, 0.0, 9.0), (2, 3.0, 4.0)]), Some(4.0));
assert_eq!(baseline(&[(1, 0.0, -0.0), (2, 3.0, 0.0)]), Some(0.0));
}
#[test]
fn parsed_metafiles_are_kept_only_within_the_cache_budget() {
let mut d = Document::new();
let media = d.add_media(red_right_half_emf(), "emf");
let lay = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default());
let outlined = HashSet::new();
let mut ex = Exporter {
doc: &d,
lay: &lay,
opts: &PdfOptions::default(),
out_index: HashMap::new(),
fonts: HashMap::new(),
font_error: None,
outlined: &outlined,
cmaps: HashMap::new(),
images: HashMap::new(),
vectors: HashMap::new(),
vector_bytes_left: crate::exporter::MAX_CACHED_VECTOR_BYTES,
tags: Vec::new(),
tag_index: HashMap::new(),
links: Vec::new(),
bookmarks: std::cell::OnceCell::new(),
};
let v = ex.vector(&media).expect("parses");
assert!(v.heap_bytes() > 0);
assert_eq!(ex.vectors.len(), 1, "kept while the budget has room");
assert_eq!(ex.vector_bytes_left, crate::exporter::MAX_CACHED_VECTOR_BYTES - v.heap_bytes());
// With the budget spent, a metafile is still drawn, just not kept.
ex.vectors.clear();
ex.vector_bytes_left = 0;
assert!(ex.vector(&media).is_some());
assert!(ex.vectors.is_empty());
}
+1 -1
View File
File diff suppressed because one or more lines are too long