Merge PR #2101: File: recovery copies and .json files store pictures as base64 (#586 follow-up)

ROADMAP.md: status line, progress log and revision history carry both sides.
docs/file-format-parity.md: status line and change log carry both sides.

Cargo.lock

@@ -5193,6 +5193,7 @@ version = "0.9.0"
 name = "wordcraft-doc"
 version = "0.9.0"
 dependencies = [
+ "base64",
  "proptest",
  "serde",
  "serde_json",

crates/doc/Cargo.toml

@@ -10,6 +10,7 @@ workspace = true
 
 [dependencies]
 wordcraft-geom = { workspace = true }
+base64 = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }

crates/doc/src/byte_store.rs

@@ -0,0 +1,130 @@
+//! How [`crate::Document::media`] and [`crate::Document::passthrough`] are written in the native
+//! JSON form (`.json` files and crash-recovery copies): each part's bytes as one base64 string.
+//! serde's default for bytes, an array of numbers, took about 3.6 times their size compact and
+//! over six times pretty-printed (a 27 MB picture made a 99 MB recovery copy, #586).
+//!
+//! Reading takes either form, so `.json` files and recovery copies written before still open.
+use std::collections::BTreeMap;
+use std::fmt;
+use std::sync::Arc;
+
+use base64::Engine as _;
+use base64::display::Base64Display;
+use base64::engine::general_purpose::STANDARD;
+use serde::de::{self, Deserializer, MapAccess, SeqAccess, Visitor};
+use serde::ser::{Serialize, Serializer};
+
+type Store = BTreeMap<String, Arc<Vec<u8>>>;
+
+/// One part's bytes, encoded as it is written out: no second copy of a large picture is made.
+struct Encoded<'a>(&'a [u8]);
+
+impl Serialize for Encoded<'_> {
+    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+        s.collect_str(&Base64Display::new(self.0, &STANDARD))
+    }
+}
+
+pub(crate) fn serialize<S: Serializer>(store: &Store, s: S) -> Result<S::Ok, S::Error> {
+    s.collect_map(store.iter().map(|(name, bytes)| (name, Encoded(bytes))))
+}
+
+pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Store, D::Error> {
+    d.deserialize_map(StoreVisitor)
+}
+
+struct StoreVisitor;
+
+impl<'de> Visitor<'de> for StoreVisitor {
+    type Value = Store;
+
+    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("a map of part names to base64 text or byte arrays")
+    }
+
+    fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Store, A::Error> {
+        let mut store = Store::new();
+        while let Some((name, Bytes(bytes))) = map.next_entry::<String, Bytes>()? {
+            store.insert(name, Arc::new(bytes));
+        }
+        Ok(store)
+    }
+}
+
+/// A part's bytes as either form.
+struct Bytes(Vec<u8>);
+
+impl<'de> de::Deserialize<'de> for Bytes {
+    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
+        d.deserialize_any(BytesVisitor).map(Bytes)
+    }
+}
+
+struct BytesVisitor;
+
+impl<'de> Visitor<'de> for BytesVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("base64 text or an array of bytes")
+    }
+
+    fn visit_str<E: de::Error>(self, text: &str) -> Result<Vec<u8>, E> {
+        STANDARD.decode(text).map_err(|e| E::custom(format!("bytes are not valid base64: {e}")))
+    }
+
+    fn visit_bytes<E: de::Error>(self, bytes: &[u8]) -> Result<Vec<u8>, E> {
+        Ok(bytes.to_vec())
+    }
+
+    /// The form written before: one number per byte.
+    fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<u8>, A::Error> {
+        // The length hint comes from the file: trust it only so far.
+        let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(1 << 20));
+        while let Some(b) = seq.next_element::<u8>()? {
+            out.push(b);
+        }
+        Ok(out)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug, Default)]
+    struct Parts {
+        #[serde(default, with = "super")]
+        media: Store,
+    }
+
+    fn parts(entries: &[(&str, &[u8])]) -> Parts {
+        Parts { media: entries.iter().map(|(n, b)| ((*n).to_string(), Arc::new(b.to_vec()))).collect() }
+    }
+
+    #[test]
+    fn bytes_are_written_as_base64_and_read_back() {
+        let all: Vec<u8> = (0..=255).collect();
+        for n in [0, 1, 2, 3, 4, 255, 256] {
+            let p = parts(&[("a.png", &all[..n.min(all.len())])]);
+            let text = serde_json::to_string(&p).unwrap();
+            assert_eq!(serde_json::from_str::<Parts>(&text).unwrap(), p, "{n}");
+        }
+        assert_eq!(serde_json::to_string(&parts(&[("m", b"Man"), ("ma", b"Ma")])).unwrap(), r#"{"media":{"m":"TWFu","ma":"TWE="}}"#);
+    }
+
+    #[test]
+    fn number_arrays_written_before_still_read() {
+        let old = r#"{"media":{"a.png":[0,1,127,128,254,255],"empty":[]}}"#;
+        assert_eq!(serde_json::from_str::<Parts>(old).unwrap(), parts(&[("a.png", &[0, 1, 127, 128, 254, 255]), ("empty", &[])]));
+    }
+
+    #[test]
+    fn hostile_values_are_errors_not_panics() {
+        for bad in [r#""T""#, r#""TWFu!""#, r#""-_""#, r#""TQ=x""#, "[256]", "[-1]", "[1.5]", "true", "{}", r#"["a"]"#] {
+            let text = format!(r#"{{"media":{{"a.png":{bad}}}}}"#);
+            assert!(serde_json::from_str::<Parts>(&text).is_err(), "{bad}");
+        }
+        assert!(serde_json::from_str::<Parts>(r#"{"media":"x"}"#).is_err());
+    }
+}

crates/doc/src/lib.rs

@@ -11,6 +11,7 @@
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 
 pub mod bidi;
+mod byte_store;
 pub mod chart;
 mod cjk_numbers;
 pub mod cluster;
@@ -723,11 +724,11 @@ pub struct Document {
     /// The document's own endnote separators and continuation notice.
     #[serde(skip_serializing_if = "NoteSeparators::is_empty")]
     pub endnote_separators: NoteSeparators,
-    /// Embedded media (images) by key.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    /// Embedded media (images) by key. Native JSON writes each as base64 (`byte_store.rs`).
+    #[serde(skip_serializing_if = "BTreeMap::is_empty", with = "byte_store")]
     pub media: BTreeMap<String, Arc<Vec<u8>>>,
     /// Source-format parts we keep verbatim for round-trip (e.g. custom XML), by part name.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(skip_serializing_if = "BTreeMap::is_empty", with = "byte_store")]
     pub passthrough: BTreeMap<String, Arc<Vec<u8>>>,
     /// The document's own building blocks (its glossary document), in file order.
     #[serde(skip_serializing_if = "Vec::is_empty")]

crates/engine/src/recovery/desktop.rs

@@ -706,6 +706,26 @@ mod tests {
         assert_eq!(Store::files(&t.0.join(id)).unwrap().len(), 1);
     }
 
+    /// Copies are written with picture bytes in base64 (#586 follow-up); those written before kept
+    /// them as arrays of numbers and are still listed and restored, bytes intact.
+    #[test]
+    fn copies_with_byte_arrays_written_before_still_restore() {
+        let t = Temp::new();
+        let id = new_id().unwrap();
+        let mut s = snapshot(&id, "Synthetic draft with a picture");
+        s.generation = 1;
+        s.document.media.insert("media/image1.png".into(), Arc::new(vec![0, 1, 127, 128, 254, 255]));
+        let mut value = serde_json::to_value(&s).unwrap();
+        assert_eq!(value["document"]["media"]["media/image1.png"], "AAF/gP7/");
+        value["document"]["media"]["media/image1.png"] = json!([0, 1, 127, 128, 254, 255]);
+        private_dir(&t.0.join(&id)).unwrap();
+        fs::write(t.0.join(&id).join("snapshot-00000000000000000001.json"), serde_json::to_vec(&value).unwrap()).unwrap();
+        let mut store = Store::new(t.0.clone()).unwrap();
+        store.scan().unwrap();
+        assert_eq!(store.inventory().len(), 1);
+        assert_eq!(store.load(&id).unwrap().document, s.document);
+    }
+
     #[test]
     fn failed_pruning_cannot_push_a_record_beyond_the_discovery_limit() {
         let t = Temp::new();

crates/engine/tests/recovery_format.rs

@@ -67,3 +67,88 @@ fn existing_native_json_without_byte_stores_still_opens() {
     let doc = wordcraft_engine::io::open_bytes("old.json", &serde_json::to_vec(&value).unwrap()).unwrap();
     assert_eq!(doc.plain_text(wordcraft_doc::StoryRef::Body), "Old native document");
 }
+
+/// Picture and part bytes are one base64 string each in native JSON (`.json` files and recovery
+/// copies, #586). An array of numbers took about 3.6 times their size, and over ten times in a
+/// pretty-printed `.json` file.
+#[test]
+fn native_json_stores_bytes_as_base64() {
+    let mut doc = Document::from_text("Synthetic");
+    let compact = |d: &Document| serde_json::to_vec(d).unwrap().len();
+    let pretty = |d: &Document| wordcraft_engine::io::save_bytes("doc.json", d).unwrap();
+    let (without, without_pretty) = (compact(&doc), pretty(&doc).len());
+    let bytes: Vec<u8> = (0..300_000u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8).collect();
+    doc.media.insert("big.png".into(), Arc::new(bytes.clone()));
+    doc.passthrough.insert("customXml/item1.xml".into(), Arc::new(b"<synthetic/>".to_vec()));
+    let saved = pretty(&doc);
+    // Base64 is 4/3 of the bytes; allow a little for names and punctuation.
+    for (with, before) in [(compact(&doc), without), (saved.len(), without_pretty)] {
+        assert!(with - before < bytes.len() * 14 / 10, "{} bytes for {}", with - before, bytes.len());
+    }
+    let value: serde_json::Value = serde_json::from_slice(&saved).unwrap();
+    assert_eq!(value["passthrough"]["customXml/item1.xml"], "PHN5bnRoZXRpYy8+");
+    let reopened = wordcraft_engine::io::open_bytes("doc.json", &saved).unwrap();
+    assert_eq!((reopened.media, reopened.passthrough), (doc.media, doc.passthrough));
+}
+
+/// `.json` files and recovery copies written before kept bytes as arrays of numbers: they still
+/// open, with the same bytes.
+#[test]
+fn native_json_with_byte_arrays_still_opens() {
+    let doc = complete_document();
+    let mut value = serde_json::to_value(&doc).unwrap();
+    for (store, parts) in [("media", &doc.media), ("passthrough", &doc.passthrough)] {
+        for (name, bytes) in parts {
+            value[store][name] = serde_json::json!(bytes.as_slice());
+        }
+    }
+    assert!(value["media"]["synthetic.png"].is_array());
+    let reopened = wordcraft_engine::io::open_bytes("old.json", &serde_json::to_vec_pretty(&value).unwrap()).unwrap();
+    assert_eq!(reopened, doc);
+}
+
+/// Bytes that are neither base64 nor an array of bytes make the file fail to open, never panic.
+#[test]
+fn native_json_with_bad_bytes_is_an_error() {
+    let saved = wordcraft_engine::io::save_bytes("doc.json", &complete_document()).unwrap();
+    for bad in [serde_json::json!("not base64!"), serde_json::json!([1, 2, 300]), serde_json::json!({"a": 1}), serde_json::json!(7)] {
+        let mut value: serde_json::Value = serde_json::from_slice(&saved).unwrap();
+        value["media"]["synthetic.png"] = bad.clone();
+        assert!(wordcraft_engine::io::open_bytes("doc.json", &serde_json::to_vec(&value).unwrap()).is_err(), "{bad}");
+    }
+}
+
+/// Measures native JSON with a large picture: the compact form a recovery copy wraps, and a
+/// pretty-printed `.json` file. Run on demand:
+/// `cargo test --release -p wordcraft-engine --test recovery_format -- --ignored --nocapture`.
+#[test]
+#[ignore = "measurement"]
+fn native_json_size_with_a_large_picture() {
+    let mut doc = Document::from_text("Synthetic document with one large picture.");
+    // Incompressible bytes, as a photograph's are.
+    let mut x = 0x2545_f491_4f6c_dd1du64;
+    let noise: Vec<u8> = (0..27u32 << 20)
+        .map(|_| {
+            x ^= x << 13;
+            x ^= x >> 7;
+            x ^= x << 17;
+            x as u8
+        })
+        .collect();
+    doc.media.insert("media/image1.png".into(), Arc::new(noise));
+    let mib = |n: usize| n as f64 / f64::from(1u32 << 20);
+    let t0 = std::time::Instant::now();
+    let compact = serde_json::to_vec(&doc).unwrap();
+    let written = t0.elapsed();
+    let t0 = std::time::Instant::now();
+    let back: Document = serde_json::from_slice(&compact).unwrap();
+    println!("27 MiB picture, compact (recovery copy): {:.1} MiB, written in {written:?}, read in {:?}", mib(compact.len()), t0.elapsed());
+    assert_eq!(back.media, doc.media);
+    let t0 = std::time::Instant::now();
+    let pretty = wordcraft_engine::io::save_bytes("doc.json", &doc).unwrap();
+    let written = t0.elapsed();
+    let t0 = std::time::Instant::now();
+    let back = wordcraft_engine::io::open_bytes("doc.json", &pretty).unwrap();
+    println!("27 MiB picture, .json file (pretty): {:.1} MiB, written in {written:?}, read in {:?}", mib(pretty.len()), t0.elapsed());
+    assert_eq!(back.media, doc.media);
+}

docs/file-format-parity.md

@@ -1,6 +1,6 @@
 # File-format parity with Microsoft Word
 
-> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-19 · **Change:** trivial (ODT headers and footers read from each master page, #863; previously trivial: shape gradients and transparency, #1500; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `<blockquote>` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats
+> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-19 · **Change:** trivial (native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: shape gradients and transparency, #1500; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `<blockquote>` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats
 
 Every format Word opens or saves, what WordCraft does with it, and how it's tested. Fidelity is the
 share of a typical real-world file's content and formatting that survives (read: shows correctly;
@@ -267,6 +267,7 @@ Imported text fields use the first nonempty cached result run's formatting on th
 | 2026-10-16 | trivial | DOCX: `wp:anchor/@layoutInCell="0"` and VML `o:allowincell="f"` are read and written back (`layoutInCell="0"`), so a header-cell picture watermark is placed on the page (#1775) |
 | 2026-10-16 | trivial | Equations from Word files no longer lose content: body-level `m:oMathPara`, `mc:AlternateContent Requires="m"`, hyperlinks/simple fields/`mc:AlternateContent` inside an equation, `w:sym` and Symbol-font text in math (#1273) |
 | 2026-10-13 | trivial | Recipient lists: cell text capped at 64 MiB a sheet (shared-string amplification), an XML spreadsheet named .xls explained (#1287) |
+| 2026-10-15 | trivial | Native .json writes pictures and kept parts as base64 text instead of arrays of numbers (a 27 MiB picture: 36 MiB instead of 286 MiB); files with the old arrays still open (#586 follow-up) |
 | 2026-10-13 | trivial | A cell's explicit top vertical alignment (`w:vAlign w:val="top"`) is kept on read and write and beats a table style that centres or bottom-aligns (#1288) |
 | 2026-10-14 | trivial | Row table border exceptions survive DOCX saves |
 | 2026-10-12 | trivial | Fields keep `w:dirty` through DOCX round trips — on buffered fields and range-field begin markers alike — and computing a result (Update Field, Update Table) clears it like Word's F9, so a generator's placeholder TOC still refreshes when Word opens the file until WordCraft replaces the placeholder (#877) |

docs/recovery.md

@@ -32,7 +32,9 @@ be lost if a crash happens before the next snapshot or while its write is still
 
 Version 1 snapshots wrap the native JSON document with an id, document name, timestamp and
 monotonic generation. Native JSON now includes media bytes, opaque preserved parts, and
-chart/SmartArt/OLE source markup and relationships. Older JSON without these fields still opens.
+chart/SmartArt/OLE source markup and relationships. Media and preserved parts are written as one
+base64 string each. Older JSON without these fields, or with their bytes as arrays of numbers (as
+written before base64), still opens.
 Paragraph cache revisions are transient and regenerated on load; undo history and selection are
 not part of the document snapshot. Recovery preserves the current in-memory model. It cannot
 recover content already lost during import or make later export more faithful than the exporter.
@@ -50,8 +52,9 @@ Windows is therefore unverified.
 Limits are 256 MiB of encoded JSON per snapshot, 2 GiB of snapshot files per profile, 128 records
 and 16 generations per record during discovery. Ordinarily only two valid generations remain.
 Copies are retained until saved or explicitly discarded; there is no automatic age-based deletion.
-Over-limit writes report an error and retain the previous valid copy. JSON stores binary parts
-as byte arrays, so it can be substantially larger than DOCX. This first version has a fixed
+Over-limit writes report an error and retain the previous valid copy. Base64 takes 4/3 of a
+part's size (a 27 MiB picture makes a 36 MiB copy), and DOCX compresses, so a copy can still be
+larger than the DOCX it came from. This first version has a fixed
 interval, English fallback for new labels, and no cloud, version-history or browser recovery.
 
 ## Commands and verification