From 02072c310aae178a27d9ada29995c2986238faaf Mon Sep 17 00:00:00 2001 From: haven <54509200+Haven-King@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:49:28 -0700 Subject: [PATCH] Format v4: large data after the JSON --- crates/format/src/lib.rs | 104 ++++++++++++++++++++++++++----- crates/format/tests/native_v3.rs | 2 +- crates/format/tests/native_v4.rs | 78 +++++++++++++++++++++++ 3 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 crates/format/tests/native_v4.rs diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 5407240c1..c6395c5c1 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -23,7 +23,10 @@ //! Versions: v1 wrote anchors as `{p: {x, y}, in: {x, y}, out: {x, y}, kind}`; v2 as //! `{p: [x, y], in?, out?, kind?}` with default-valued fields left out; v3 keeps the assigned colour //! profiles in `document.color_profiles` (before: `document.unknown.colorProfiles`) and may be -//! compressed. Every version loads, and [`save_with`] writes any of them for older apps (under the +//! compressed; v4 may keep large data (images, the PDF, profiles of at least +//! [`INLINE_MAX`] bytes) after the JSON instead of in it: the JSON is followed by [`BLOB_MAGIC`] and +//! the bytes, and such an entry says where they are (`"blob": [offset, length]`) instead of giving +//! `data`. Every version loads, and [`save_with`] writes any of them for older apps (under the //! format name `drawcraft`, which every version reads). Readers reject files whose `version` is //! newer than they support. #![forbid(unsafe_code)] @@ -42,8 +45,12 @@ use vectorcraft_doc::{Document, ImageBlob}; #[cfg(not(target_arch = "wasm32"))] pub use atomic::{write_atomic, write_atomic_with}; -/// v3: colour profiles as a document field, optional compression and preview (v1 and v2 files still load). -pub const VERSION: u32 = 3; +/// v4: large data after the JSON (v1 to v3 files still load). +pub const VERSION: u32 = 4; +/// Data this large or larger goes after the JSON (v4), not into it as base64. +pub const INLINE_MAX: usize = 8 << 10; +/// What separates the JSON from the data after it (v4). +pub const BLOB_MAGIC: &[u8] = b"\n\0VCBLOBS\0"; /// The oldest version [`save_with`] writes. pub const MIN_VERSION: u32 = 1; /// The first version whose readers open compressed files. @@ -87,12 +94,64 @@ pub enum FormatError { #[derive(Serialize, Deserialize)] struct Image { mime: String, + /// The bytes as base64, unless they come after the JSON (`blob`). + #[serde(default, skip_serializing_if = "String::is_empty")] data: String, + /// Where the bytes are in the data after the JSON (v4): `[offset, length]`. + #[serde(default, skip_serializing_if = "Option::is_none")] + blob: Option<[u64; 2]>, /// `data` is a linked image's preview ([`ImageBlob::proxy`]), not the file's bytes. #[serde(default, skip_serializing_if = "std::ops::Not::not")] proxy: bool, } +impl Image { + /// Its bytes: the base64 `data`, or its part of `tail` (the data after the JSON). + fn bytes(&self, tail: &[u8]) -> Option> { + match self.blob { + Some([offset, len]) => { + let start = usize::try_from(offset).ok()?; + let end = start.checked_add(usize::try_from(len).ok()?)?; + tail.get(start..end).map(<[u8]>::to_vec) + } + None => base64_decode(&self.data), + } + } +} + +/// Entries written for one file: large bytes go to the data after the JSON when the version keeps +/// them there. +struct Blobs { + tail: Vec, + outside: bool, +} + +impl Blobs { + fn image(&mut self, mime: &str, bytes: &[u8], proxy: bool) -> Image { + if self.outside && bytes.len() >= INLINE_MAX { + let at = self.tail.len() as u64; + self.tail.extend_from_slice(bytes); + return Image { mime: mime.into(), data: String::new(), blob: Some([at, bytes.len() as u64]), proxy }; + } + Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy } + } +} + +/// The JSON of a file and the data after it (v4), split at [`BLOB_MAGIC`] after the first JSON +/// value. +fn split_tail(text: &[u8]) -> (&[u8], &[u8]) { + let mut values = serde_json::Deserializer::from_slice(text).into_iter::(); + if values.next().is_some_and(|v| v.is_ok()) { + let end = values.byte_offset(); + if let Some(rest) = text.get(end..) + && rest.starts_with(BLOB_MAGIC) + { + return (text.get(..end).unwrap_or(text), rest.get(BLOB_MAGIC.len()..).unwrap_or_default()); + } + } + (text, &[]) +} + #[derive(Deserialize)] struct File { format: String, @@ -189,6 +248,7 @@ pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError // Images are only referenced by image objects' `key` (wherever the objects are: layers, // symbols, patterns, masks, or foreign data). let used = if blobs.is_empty() { HashSet::new() } else { string_members(&body, "key") }; + let mut out_blobs = Blobs { tail: vec![], outside: o.version >= 4 }; let images: BTreeMap<&str, Image> = blobs .iter() .filter(|(k, _)| used.contains(k.as_str())) @@ -196,20 +256,20 @@ pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError // Include Linked Files keeps the file's bytes whenever they are loaded. let preview_only = linked.contains(k) && (!o.include_linked || b.is_proxy()); let image = match b.proxy.as_ref().filter(|_| preview_only) { - Some(p) => Image { mime: vectorcraft_doc::links::PROXY_MIME.into(), data: base64_encode(p), proxy: true }, - None => Image { mime: b.mime.clone(), data: base64_encode(&b.bytes), proxy: false }, + Some(p) => out_blobs.image(vectorcraft_doc::links::PROXY_MIME, p, true), + None => out_blobs.image(&b.mime, &b.bytes, false), }; (k.as_str(), image) }) .collect(); - let blob = |mime: &str, bytes: &[u8]| Image { mime: mime.into(), data: base64_encode(bytes), proxy: false }; - let profiles: BTreeMap<&str, Image> = o.profiles.iter().map(|(name, icc)| (name.as_str(), blob(ICC_MIME, icc))).collect(); - let pdf = o.pdf.as_deref().map(|p| blob("application/pdf", p)); + let profiles: BTreeMap<&str, Image> = o.profiles.iter().map(|(name, icc)| (name.as_str(), out_blobs.image(ICC_MIME, icc, false))).collect(); + let pdf = o.pdf.as_deref().map(|p| out_blobs.image("application/pdf", p, false)); + let blob = |mime: &str, bytes: &[u8]| Image { mime: mime.into(), data: base64_encode(bytes), blob: None, proxy: false }; let carried = profiles.values().chain(&pdf).map(|i| i.data.len()).sum::(); let size = body.len() + images.values().map(|i| i.data.len()).sum::() + carried + 256; let mut w = Envelope { out: Vec::with_capacity(size), pretty }; // Older apps only know the name from before the rename. - w.field("format", &if o.version < VERSION { LEGACY_EXTENSION } else { EXTENSION })?; + w.field("format", &if o.version < 3 { LEGACY_EXTENSION } else { EXTENSION })?; w.field("version", &o.version)?; w.field("generator", &format!("VectorCraft {}", env!("CARGO_PKG_VERSION")))?; if let Some(png) = &o.preview { @@ -223,7 +283,11 @@ pub fn save_with(doc: &Document, o: &SaveOptions) -> Result, FormatError } w.raw("document", &body); w.field("images", &images)?; - let out = w.finish(); + let mut out = w.finish(); + if !out_blobs.tail.is_empty() { + out.extend_from_slice(BLOB_MAGIC); + out.extend_from_slice(&out_blobs.tail); + } if !o.compress { return Ok(out); } @@ -356,7 +420,8 @@ pub fn load_info(bytes: &[u8]) -> Result<(Document, FileInfo), FormatError> { /// [`load`], with everything else the file carries. pub fn load_file(bytes: &[u8]) -> Result { let text = unpack(bytes)?; - let f: File = serde_json::from_slice(&text).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; + let (json, tail) = split_tail(&text); + let f: File = serde_json::from_slice(json).map_err(|e| FormatError::NotVectorcraft(e.to_string()))?; if f.format != EXTENSION && f.format != LEGACY_EXTENSION { return Err(FormatError::NotVectorcraft(format!("format is `{}`", f.format))); } @@ -365,7 +430,7 @@ pub fn load_file(bytes: &[u8]) -> Result { } let mut doc = f.document; for (k, img) in f.images { - let bytes = base64_decode(&img.data).ok_or_else(|| FormatError::BadImage(k.clone()))?; + let bytes = img.bytes(tail).ok_or_else(|| FormatError::BadImage(k.clone()))?; let mut blob = ImageBlob::new(img.mime, bytes); // Until the linked file is read, the preview stands in for it. if img.proxy { @@ -382,7 +447,7 @@ pub fn load_file(bytes: &[u8]) -> Result { doc.fix_next_id(); // Untrusted: capped, named and pointing at objects the file has. doc.tidy_saved_selections(); - let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, base64_decode(&icc.data)?))).collect(); + let profiles = f.profiles.into_iter().filter_map(|(name, icc)| Some((name, icc.bytes(tail)?))).collect(); Ok(NativeFile { doc, info: FileInfo { version: f.version, legacy: f.format == LEGACY_EXTENSION }, profiles }) } @@ -393,19 +458,24 @@ struct Head { pdf: Option, } -fn head(bytes: &[u8]) -> Option { - serde_json::from_slice(&unpack(bytes).ok()?).ok() +/// The file's head and the data after its JSON. +fn head(bytes: &[u8]) -> Option<(Head, Vec)> { + let text = unpack(bytes).ok()?; + let (json, tail) = split_tail(&text); + Some((serde_json::from_slice(json).ok()?, tail.to_vec())) } /// The preview PNG embedded in a native file (`None`: it has none, or isn't a native file). pub fn preview(bytes: &[u8]) -> Option> { - base64_decode(&head(bytes)?.preview?.data) + let (h, tail) = head(bytes)?; + h.preview?.bytes(&tail) } /// The PDF a native file saved with Create PDF-Compatible File carries (`None`: it has none, or /// isn't a native file). pub fn pdf_content(bytes: &[u8]) -> Option> { - base64_decode(&head(bytes)?.pdf?.data) + let (h, tail) = head(bytes)?; + h.pdf?.bytes(&tail) } /// Does this look like a `.vectorcraft` file (compressed or not)? diff --git a/crates/format/tests/native_v3.rs b/crates/format/tests/native_v3.rs index cde12f0d0..8ca0d21d4 100644 --- a/crates/format/tests/native_v3.rs +++ b/crates/format/tests/native_v3.rs @@ -55,7 +55,7 @@ fn compressed_files_load() { assert!(!flate2_unpack(&packed).contains(&b'\n')); // A legacy (v2, pre-rename) file compressed by hand opens too. let text = String::from_utf8(plain.clone()).unwrap(); - let legacy = text.replacen("\"format\":\"vectorcraft\",\"version\":3", "\"format\":\"drawcraft\",\"version\":2", 1); + let legacy = text.replacen(&format!("\"format\":\"vectorcraft\",\"version\":{VERSION}"), "\"format\":\"drawcraft\",\"version\":2", 1); assert_ne!(legacy, text); let legacy = gzip(legacy.as_bytes()); assert!(sniff(&legacy)); diff --git a/crates/format/tests/native_v4.rs b/crates/format/tests/native_v4.rs new file mode 100644 index 000000000..34f6c6a8b --- /dev/null +++ b/crates/format/tests/native_v4.rs @@ -0,0 +1,78 @@ +//! `.vectorcraft` v4: large data after the JSON (images, the PDF, profiles), small files still plain +//! JSON, older versions inline, damaged offsets refused. +// Integration tests: unwrapping and panicking on failure is fine here, unlike in shipped code (AGENTS.md › Robustness). +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::sync::Arc; + +use serde_json::Value; +use vectorcraft_doc::{Document, ImageBlob, ImageObject, Node, NodeKind}; +use vectorcraft_format::{BLOB_MAGIC, FormatError, INLINE_MAX, SaveOptions, load, load_file, pdf_content, save_with}; +use vectorcraft_geom::Affine; + +/// A document showing image `key` of `len` bytes. +fn doc_with_image(key: &str, len: usize) -> Document { + let mut d = Document::new(100.0, 100.0); + let bytes: Vec = (0..len).map(|i| (i * 7 % 251) as u8).collect(); + d.images.insert(key.into(), ImageBlob::new("image/png", bytes)); + let l = d.layers[0].id; + let id = d.alloc_id(); + let im = ImageObject { key: key.into(), width: 10, height: 10, xf: Affine::IDENTITY, link: None, placement: Default::default() }; + d.insert(Some(l), 0, Node::new(id, NodeKind::Image(im))).unwrap(); + d +} + +fn has_tail(bytes: &[u8]) -> bool { + bytes.windows(BLOB_MAGIC.len()).any(|w| w == BLOB_MAGIC) +} + +#[test] +fn large_data_goes_after_the_json_and_reads_back() { + let d = doc_with_image("big", INLINE_MAX + 10); + let pdf = vec![b'%'; INLINE_MAX * 2]; + for compress in [false, true] { + let o = SaveOptions { compress, pdf: Some(pdf.clone()), ..SaveOptions::default() }; + let bytes = save_with(&d, &o).unwrap(); + if !compress { + assert!(has_tail(&bytes), "a binary section"); + // The JSON before it holds no base64 copy of the image. + assert!(bytes.len() < INLINE_MAX + 10 + pdf.len() + (32 << 10), "{} bytes", bytes.len()); + } + let back = load(&bytes).unwrap(); + assert_eq!(back.images["big"].bytes, d.images["big"].bytes, "compress {compress}"); + assert_eq!(pdf_content(&bytes).unwrap(), pdf); + } +} + +#[test] +fn small_files_stay_plain_json_and_older_versions_inline() { + let small = save_with(&doc_with_image("small", 100), &SaveOptions::default()).unwrap(); + assert!(!has_tail(&small)); + let v: Value = serde_json::from_slice(&small).unwrap(); + assert_eq!(v["version"], 4); + let big = doc_with_image("big", INLINE_MAX + 10); + let v3 = save_with(&big, &SaveOptions { version: 3, ..SaveOptions::default() }).unwrap(); + assert!(!has_tail(&v3), "version 3 keeps everything in the JSON"); + let v: Value = serde_json::from_slice(&v3).unwrap(); + assert_eq!((v["format"].as_str(), v["version"].as_u64()), (Some("vectorcraft"), Some(3))); + assert_eq!(load(&v3).unwrap().images["big"].bytes, big.images["big"].bytes); +} + +#[test] +fn damaged_offsets_are_refused_not_a_crash() { + let d = doc_with_image("big", INLINE_MAX + 10); + let bytes = save_with(&d, &SaveOptions::default()).unwrap(); + // The data cut short. + let cut = &bytes[..bytes.len() - 100]; + assert!(matches!(load_file(cut), Err(FormatError::BadImage(_)))); + // An offset past the end. + let text = String::from_utf8_lossy(&bytes); + let at = text.find("\"blob\":[").unwrap(); + let mut damaged = bytes.clone(); + damaged.splice(at..at + 8, b"\"blob\":[9".iter().copied()); + assert!(load_file(&damaged).is_err()); + // A missing section. + let json_only = &bytes[..bytes.windows(BLOB_MAGIC.len()).position(|w| w == BLOB_MAGIC).unwrap()]; + assert!(load_file(json_only).is_err()); + let _ = Arc::new(()); +}