mirror of
https://github.com/storytold/deckcraft.git
synced 2026-11-03 08:50:47 -06:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b83c712da | ||
|
|
f7fe74cecd |
+38
-161
@@ -37,146 +37,56 @@ pub struct ShapeParams<'a> {
|
||||
pub language: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Most bytes of text given to the shaper at once. The shaper lets one call grow its buffers to
|
||||
/// 256 glyphs per character (a font's substitutions decide how many), so a long item is shaped in
|
||||
/// pieces of this size, each with the rest of the paragraph as context.
|
||||
const MAX_SHAPE_CHUNK: usize = 1024;
|
||||
/// Most glyphs kept per character of a piece: a piece the font expands further is laid out one
|
||||
/// glyph per character from the cmap instead.
|
||||
const MAX_GLYPHS_PER_CHAR: usize = 32;
|
||||
|
||||
/// Shape `para[range]` with `face`, the rest of `para` as context. Glyphs are in visual order
|
||||
/// (left to right) and clusters are byte offsets into `para`. `map` substitutes characters (All
|
||||
/// Caps) without moving clusters. An empty or out-of-bounds range shapes nothing.
|
||||
pub fn shape_range(face: &FontFace, para: &str, range: Range<usize>, params: &ShapeParams, map: impl Fn(char) -> char) -> Vec<ShapedGlyph> {
|
||||
let (Some(_), Some(text), Some(_)) = (para.get(..range.start), para.get(range.clone()), para.get(range.end..)) else {
|
||||
let (Some(pre), Some(text), Some(post)) = (para.get(..range.start), para.get(range.clone()), para.get(range.end..)) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let chunks = chunks(text, MAX_SHAPE_CHUNK);
|
||||
let mut pieces: Vec<Vec<ShapedGlyph>> = Vec::with_capacity(chunks.len());
|
||||
let mut out = Vec::with_capacity(text.len());
|
||||
let shaped = face.hb().map(|hb| {
|
||||
let shaper = face.shaper.shaper(&hb).instance(face.instance.as_ref()).build();
|
||||
// Pieces of one item share the script the whole item would be guessed to have.
|
||||
let item_script = (chunks.len() > 1 && params.script.is_none()).then(|| {
|
||||
let mut probe = UnicodeBuffer::new();
|
||||
for c in text.chars() {
|
||||
probe.add(map(c), 0);
|
||||
}
|
||||
probe.guess_segment_properties();
|
||||
probe.script()
|
||||
});
|
||||
for c in &chunks {
|
||||
let r = (range.start + c.start)..(range.start + c.end);
|
||||
let (Some(pre), Some(piece), Some(post)) = (para.get(..r.start), para.get(r.clone()), para.get(r.end..)) else { continue };
|
||||
let mut buf = UnicodeBuffer::new();
|
||||
buf.set_pre_context(pre);
|
||||
buf.set_post_context(post);
|
||||
for (i, c) in piece.char_indices() {
|
||||
buf.add(map(c), (r.start + i) as u32);
|
||||
}
|
||||
buf.guess_segment_properties();
|
||||
if let Some(s) = params.script.and_then(|t| Script::from_iso15924_tag(Tag::new(&t))) {
|
||||
buf.set_script(s);
|
||||
} else if let Some(s) = item_script {
|
||||
buf.set_script(s);
|
||||
}
|
||||
if let Some(l) =
|
||||
params.language.or_else(|| params.script.and_then(crate::script::default_language)).and_then(|l| l.parse::<Language>().ok())
|
||||
{
|
||||
buf.set_language(l);
|
||||
}
|
||||
buf.set_direction(if params.rtl { Direction::RightToLeft } else { Direction::LeftToRight });
|
||||
let gb = shaper.shape(buf, ShapeOptions::new().features(params.features));
|
||||
let budget = piece.chars().count().saturating_mul(MAX_GLYPHS_PER_CHAR);
|
||||
if gb.len() > budget {
|
||||
// Font-controlled expansion past anything real text needs: keep it bounded.
|
||||
pieces.push(cmap_glyphs(face, para, r, params.rtl, &map));
|
||||
continue;
|
||||
}
|
||||
pieces.push(
|
||||
gb.glyph_infos()
|
||||
.iter()
|
||||
.zip(gb.glyph_positions())
|
||||
.map(|(info, pos)| ShapedGlyph {
|
||||
gid: info.glyph_id,
|
||||
cluster: info.cluster as usize,
|
||||
x_advance: pos.x_advance,
|
||||
x_offset: pos.x_offset,
|
||||
y_offset: pos.y_offset,
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mut buf = UnicodeBuffer::new();
|
||||
buf.set_pre_context(pre);
|
||||
buf.set_post_context(post);
|
||||
for (i, c) in text.char_indices() {
|
||||
buf.add(map(c), (range.start + i) as u32);
|
||||
}
|
||||
buf.guess_segment_properties();
|
||||
if let Some(s) = params.script.and_then(|t| Script::from_iso15924_tag(Tag::new(&t))) {
|
||||
buf.set_script(s);
|
||||
}
|
||||
if let Some(l) = params.language.or_else(|| params.script.and_then(crate::script::default_language)).and_then(|l| l.parse::<Language>().ok())
|
||||
{
|
||||
buf.set_language(l);
|
||||
}
|
||||
buf.set_direction(if params.rtl { Direction::RightToLeft } else { Direction::LeftToRight });
|
||||
let gb = shaper.shape(buf, ShapeOptions::new().features(params.features));
|
||||
for (info, pos) in gb.glyph_infos().iter().zip(gb.glyph_positions()) {
|
||||
out.push(ShapedGlyph {
|
||||
gid: info.glyph_id,
|
||||
cluster: info.cluster as usize,
|
||||
x_advance: pos.x_advance,
|
||||
x_offset: pos.x_offset,
|
||||
y_offset: pos.y_offset,
|
||||
});
|
||||
}
|
||||
});
|
||||
if shaped.is_none() {
|
||||
if shaped.is_none()
|
||||
&& let Some(f) = face.skrifa()
|
||||
{
|
||||
// No shaper for this face: one glyph per character from the cmap, in visual order.
|
||||
return cmap_glyphs(face, para, range, params.rtl, &map);
|
||||
}
|
||||
// Each piece is in visual order; right to left, the later pieces come first.
|
||||
if params.rtl {
|
||||
pieces.reverse();
|
||||
}
|
||||
let mut out = Vec::with_capacity(pieces.iter().map(Vec::len).sum());
|
||||
for p in pieces {
|
||||
out.extend(p);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One glyph per character of `para[range]` from the cmap, in visual order.
|
||||
fn cmap_glyphs(face: &FontFace, para: &str, range: Range<usize>, rtl: bool, map: &impl Fn(char) -> char) -> Vec<ShapedGlyph> {
|
||||
let (Some(text), Some(f)) = (para.get(range.clone()), face.skrifa()) else { return Vec::new() };
|
||||
let cmap = f.charmap();
|
||||
let gm = f.glyph_metrics(Size::unscaled(), face.location());
|
||||
let mut out = Vec::with_capacity(text.len());
|
||||
for (i, c) in text.char_indices() {
|
||||
let g = cmap.map(map(c)).unwrap_or_default();
|
||||
let adv = gm.advance_width(g).unwrap_or(face.upem as f32 * 0.5);
|
||||
out.push(ShapedGlyph { gid: g.to_u32(), cluster: range.start + i, x_advance: adv.round() as i32, x_offset: 0, y_offset: 0 });
|
||||
}
|
||||
if rtl {
|
||||
out.reverse();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `text` cut into byte ranges of at most `max` bytes (or one character, if longer), at the last
|
||||
/// space of a piece where there is one, else before a character that starts a cluster (not a
|
||||
/// combining mark or joiner, not after a joiner). Text that fits is one piece.
|
||||
fn chunks(text: &str, max: usize) -> Vec<Range<usize>> {
|
||||
use unicode_bidi::BidiClass::{BN, NSM};
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
while text.len() - start > max {
|
||||
let window = text.get(start..).unwrap_or_default();
|
||||
let mut space = None;
|
||||
let mut boundary = None;
|
||||
let mut any = None;
|
||||
let mut prev: Option<char> = None;
|
||||
for (i, c) in window.char_indices() {
|
||||
if i > max {
|
||||
break;
|
||||
}
|
||||
if i > 0 {
|
||||
any = Some(i);
|
||||
let joined = prev.is_some_and(|p| matches!(unicode_bidi::bidi_class(p), BN));
|
||||
if !joined && !matches!(unicode_bidi::bidi_class(c), NSM | BN) {
|
||||
boundary = Some(i);
|
||||
if prev.is_some_and(char::is_whitespace) {
|
||||
space = Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
prev = Some(c);
|
||||
let cmap = f.charmap();
|
||||
let gm = f.glyph_metrics(Size::unscaled(), face.location());
|
||||
for (i, c) in text.char_indices() {
|
||||
let g = cmap.map(map(c)).unwrap_or_default();
|
||||
let adv = gm.advance_width(g).unwrap_or(face.upem as f32 * 0.5);
|
||||
out.push(ShapedGlyph { gid: g.to_u32(), cluster: range.start + i, x_advance: adv.round() as i32, x_offset: 0, y_offset: 0 });
|
||||
}
|
||||
if params.rtl {
|
||||
out.reverse();
|
||||
}
|
||||
// A piece always moves forward by at least one character.
|
||||
let first = window.chars().next().map_or(window.len(), char::len_utf8);
|
||||
let cut = space.filter(|&i| i >= max / 2).or(boundary).or(any).unwrap_or(first).max(first);
|
||||
out.push(start..start + cut);
|
||||
start += cut;
|
||||
}
|
||||
if start < text.len() || out.is_empty() {
|
||||
out.push(start..text.len());
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -211,37 +121,4 @@ mod tests {
|
||||
// Not a char boundary.
|
||||
assert!(shape_range(&face, "سنة", 1..2, &ShapeParams::default(), |c| c).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_items_are_shaped_in_bounded_pieces() {
|
||||
for text in ["word ".repeat(2_000), "e\u{301}".repeat(2_000), "x".repeat(5_000), "\u{1f469}\u{200d}\u{1f4bb} ".repeat(500)] {
|
||||
let pieces = chunks(&text, MAX_SHAPE_CHUNK);
|
||||
assert_eq!(pieces.first().map(|r| r.start), Some(0));
|
||||
assert_eq!(pieces.last().map(|r| r.end), Some(text.len()));
|
||||
for w in pieces.windows(2) {
|
||||
assert_eq!(w[0].end, w[1].start);
|
||||
}
|
||||
for r in &pieces {
|
||||
assert!(!r.is_empty() && r.len() <= MAX_SHAPE_CHUNK);
|
||||
// Never before a combining mark, never after a joiner.
|
||||
let rest = &text[r.end..];
|
||||
assert!(!rest.starts_with('\u{301}'));
|
||||
assert!(!text[..r.end].ends_with('\u{200d}') || rest.is_empty());
|
||||
}
|
||||
}
|
||||
assert_eq!(chunks("short", MAX_SHAPE_CHUNK), vec![0..5]);
|
||||
assert_eq!(chunks("", MAX_SHAPE_CHUNK), vec![0..0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_items_keep_every_cluster_in_visual_order() {
|
||||
let face = FontDb::global().face(DEFAULT_FAMILY, "Regular");
|
||||
let text = "ab cd ".repeat(1_000);
|
||||
let g = shape_range(&face, &text, 0..text.len(), &ShapeParams::default(), |c| c);
|
||||
assert_eq!(g.len(), text.len());
|
||||
assert!(g.windows(2).all(|w| w[0].cluster < w[1].cluster));
|
||||
let rtl = shape_range(&face, &text, 0..text.len(), &ShapeParams { rtl: true, ..Default::default() }, |c| c);
|
||||
assert_eq!(rtl.len(), text.len());
|
||||
assert!(rtl.windows(2).all(|w| w[0].cluster > w[1].cluster));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,12 +16,6 @@ const INVALID_REQUEST: i64 = -32600;
|
||||
const METHOD_NOT_FOUND: i64 = -32601;
|
||||
const INVALID_PARAMS: i64 = -32602;
|
||||
const INTERNAL_ERROR: i64 = -32603;
|
||||
|
||||
/// Entries one batch (a JSON-RPC batch, or the `batch` tool's commands) may hold.
|
||||
pub(crate) const MAX_BATCH_ITEMS: usize = 1000;
|
||||
/// Bytes of results one batch may gather before the rest of it is refused: each result can copy
|
||||
/// much of the document (an inspection holds every note), so a batch holds at most this much.
|
||||
pub(crate) const MAX_BATCH_BYTES: usize = 64 << 20;
|
||||
const RESOURCE_NOT_FOUND: i64 = -32002;
|
||||
/// Keep request frames in step with the connected backend's reply limit.
|
||||
const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024;
|
||||
@@ -99,28 +93,8 @@ impl Server {
|
||||
// Batches were removed in 2025-06-18; still answer older clients sensibly.
|
||||
if batch.is_empty() {
|
||||
Some(error(Value::Null, INVALID_REQUEST, "empty batch"))
|
||||
} else if batch.len() > MAX_BATCH_ITEMS {
|
||||
Some(error(Value::Null, INVALID_REQUEST, format!("a batch holds at most {MAX_BATCH_ITEMS} messages")))
|
||||
} else {
|
||||
let mut used = 0usize;
|
||||
let mut replies: Vec<Value> = Vec::new();
|
||||
for m in batch {
|
||||
if used > MAX_BATCH_BYTES {
|
||||
// Not run: answer each remaining request with a short error.
|
||||
if let Some(id) = m.get("id").filter(|id| id.is_string() || id.is_number()) {
|
||||
replies.push(error(id.clone(), INTERNAL_ERROR, "the batch's replies passed 64 MiB; not run"));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(r) = self.handle(m) {
|
||||
used = used.saturating_add(r.to_string().len());
|
||||
replies.push(if used > MAX_BATCH_BYTES {
|
||||
error(r.get("id").cloned().unwrap_or(Value::Null), INTERNAL_ERROR, "the batch's replies passed 64 MiB; reply dropped")
|
||||
} else {
|
||||
r
|
||||
});
|
||||
}
|
||||
}
|
||||
let replies: Vec<Value> = batch.into_iter().filter_map(|m| self.handle(m)).collect();
|
||||
(!replies.is_empty()).then_some(Value::Array(replies))
|
||||
}
|
||||
}
|
||||
|
||||
+2
-15
@@ -357,25 +357,12 @@ pub fn call_tool(b: &mut dyn Backend, name: &str, a: &Value) -> ToolResult {
|
||||
wrap(exec(b, c, a.get("params").cloned().unwrap_or(json!({}))))
|
||||
}
|
||||
"batch" => {
|
||||
use crate::server::{MAX_BATCH_BYTES, MAX_BATCH_ITEMS};
|
||||
let keep = a.get("keepGoing").and_then(Value::as_bool).unwrap_or(false);
|
||||
let commands = a.get("commands").and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default();
|
||||
if commands.len() > MAX_BATCH_ITEMS {
|
||||
return ToolResult::error(format!("a batch holds at most {MAX_BATCH_ITEMS} commands"));
|
||||
}
|
||||
let mut out = vec![];
|
||||
let mut used = 0usize;
|
||||
for c in commands {
|
||||
for c in a.get("commands").and_then(Value::as_array).cloned().unwrap_or_default() {
|
||||
let id = c.get("command").and_then(Value::as_str).unwrap_or("");
|
||||
match exec(b, id, c.get("params").cloned().unwrap_or(json!({}))) {
|
||||
Ok(v) => {
|
||||
used = used.saturating_add(v.to_string().len());
|
||||
if used > MAX_BATCH_BYTES {
|
||||
out.push(json!({"command": id, "ok": false, "error": "the batch's results passed 64 MiB; stopped here"}));
|
||||
return ToolResult { content: vec![json!({"type": "text", "text": Value::Array(out).to_string()})], is_error: true };
|
||||
}
|
||||
out.push(json!({"command": id, "ok": true, "result": v}));
|
||||
}
|
||||
Ok(v) => out.push(json!({"command": id, "ok": true, "result": v})),
|
||||
Err(e) => {
|
||||
out.push(json!({"command": id, "ok": false, "error": e}));
|
||||
if !keep {
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
//! A batch (JSON-RPC or the `batch` tool) is bounded in entries and in the bytes of results it
|
||||
//! gathers, so a short request cannot copy a large document into memory hundreds of times.
|
||||
|
||||
use deckcraft_mcp::{Headless, Server};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn server_with_notes(bytes: usize) -> Server {
|
||||
let mut server = Server::new(Box::new(Headless::with_document()));
|
||||
let line = json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "run_command", "arguments": {"command": "slide.notes", "params": {"text": "n".repeat(bytes)}}}});
|
||||
let reply: Value = serde_json::from_str(&server.handle_line(&line.to_string()).unwrap()).unwrap();
|
||||
assert_ne!(reply["result"]["isError"], true, "{reply}");
|
||||
server
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_rpc_batches_are_bounded_in_entries_and_reply_bytes() {
|
||||
let mut server = server_with_notes(1 << 20);
|
||||
let ping = |i: usize| json!({"jsonrpc": "2.0", "id": i, "method": "ping"});
|
||||
let too_many: Vec<Value> = (0..1001).map(ping).collect();
|
||||
let reply: Value = serde_json::from_str(&server.handle_line(&Value::Array(too_many).to_string()).unwrap()).unwrap();
|
||||
assert_eq!(reply["error"]["code"], -32600, "{reply}");
|
||||
// 100 inspections of a slide with 1 MB of notes: about 64 are answered, the rest refused.
|
||||
let inspect = |i: usize| json!({"jsonrpc": "2.0", "id": i, "method": "tools/call", "params": {"name": "inspect_slide", "arguments": {}}});
|
||||
let line = Value::Array((0..100).map(inspect).collect()).to_string();
|
||||
let reply = server.handle_line(&line).unwrap();
|
||||
assert!(reply.len() < (64 << 20) + (4 << 20), "{} bytes", reply.len());
|
||||
let replies: Vec<Value> = serde_json::from_str(&reply).unwrap();
|
||||
assert_eq!(replies.len(), 100, "every request is answered");
|
||||
assert!(replies.last().unwrap()["error"]["message"].as_str().unwrap().contains("64 MiB"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_batch_tool_is_bounded_in_commands_and_result_bytes() {
|
||||
let mut server = server_with_notes(1 << 20);
|
||||
let call = |server: &mut Server, commands: Vec<Value>| {
|
||||
let line = json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "batch", "arguments": {"commands": commands, "keepGoing": true}}});
|
||||
let reply: Value = serde_json::from_str(&server.handle_line(&line.to_string()).unwrap()).unwrap();
|
||||
(reply["result"]["isError"] == true, reply["result"]["content"][0]["text"].as_str().unwrap_or_default().to_string())
|
||||
};
|
||||
let (err, text) = call(&mut server, vec![json!({"command": "document.inspect"}); 1001]);
|
||||
assert!(err && text.contains("at most 1000"), "{text}");
|
||||
let (err, text) = call(&mut server, vec![json!({"command": "slide.inspect"}); 100]);
|
||||
assert!(err && text.contains("64 MiB"), "{}", &text[text.len().saturating_sub(200)..]);
|
||||
assert!(text.len() < (64 << 20) + (4 << 20));
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
// Integration tests: unwrapping and panicking on failure is fine here, unlike in shipped code (AGENTS.md).
|
||||
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
mod automation_roots;
|
||||
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
mod batch_budget;
|
||||
|
||||
Reference in New Issue
Block a user