From ed623902442f2853d758f87906e8140bc880508d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:12:56 +0200 Subject: [PATCH 1/2] XLSX: keep unmodelled parts on save (#330) Pass-through of package content GridCraft does not model: on read the raw parts are kept with their content types and relationship graph per owner (package root, workbook, sheet); on write they get fresh part names where needed and new relationship ids, and the r:ids in kept XML are rewritten. Kept: custom document properties, ribbon customisation, add-in task panes, custom XML, links to external workbooks with (never followed), connections, sheet query tables, printer settings with the (#324), form and ActiveX controls, OLE objects and sheet backgrounds with their VML shapes (merged with GridCraft's note shapes). Content whose graph reaches parts GridCraft rewrites is dropped with a warning; sheet content goes with its sheet and is not copied by Copy Sheet. Fixes #324 Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 1 + crates/engine/src/cmd/sheet.rs | 1 + crates/model/src/lib.rs | 2 + crates/model/src/preserved.rs | 70 ++ crates/model/src/sheet.rs | 5 + crates/model/src/workbook.rs | 4 + crates/xlsx/src/lib.rs | 1 + crates/xlsx/src/passthrough.rs | 898 +++++++++++++++++++++ crates/xlsx/src/read.rs | 6 +- crates/xlsx/src/sheet_read.rs | 1 - crates/xlsx/src/sheet_write.rs | 15 +- crates/xlsx/src/tests/mod.rs | 1 + crates/xlsx/src/tests/passthrough_tests.rs | 430 ++++++++++ crates/xlsx/src/write.rs | 11 + crates/xlsx/src/xml.rs | 2 +- docs/file-format-parity.md | 7 +- docs/gaps.md | 17 +- 17 files changed, 1455 insertions(+), 17 deletions(-) create mode 100644 crates/model/src/preserved.rs create mode 100644 crates/xlsx/src/passthrough.rs create mode 100644 crates/xlsx/src/tests/passthrough_tests.rs diff --git a/ROADMAP.md b/ROADMAP.md index 1189fd721..f2c255a45 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -131,6 +131,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-10-11 | XLSX pass-through: custom XML, external links, connections, query tables, printer settings, form/ActiveX controls and OLE objects are kept on save (#330, #324) | | 2026-10-10 | Ribbon translated into Japanese, Chinese, Korean, Brazilian Portuguese and Russian with system-language default (#31, #104, #82); CJK/Thai/Hebrew font fallback (#38); clear messages for encrypted and `.xls` files (#17); smooth line charts (#62). Progress docs restructured to the craftrules standard; full re-measure (this page). Initial ribbon key tips (#43), in-place text box editing (#84), copy as HTML table (#36). ODS and XLSB data import (#100, #101); recalc linear in the number of range formulas (#66); RISC-V Linux build; Flatpak repository on Pages; web cell-commit trap fixed | | 2026-10-09 | Community fixes: spill range operator `A1#`, sheet-scoped names, protection refuses locked-cell edits, Insert/Delete Cells adjust references, `DATE(1900,2,29)`, SUMIF sizing, `ROUND` at 15 digits, `_xlpm.` LAMBDA parameters in XLSX, vertical scrollbar, system appearance, border previews; v0.4.0 | | 2026-10-08 | v0.2.0 and v0.3.0 releases; Flatpak, AppImage zsync, branded DMG; desktop log file; DirectX 12 default on Windows | diff --git a/crates/engine/src/cmd/sheet.rs b/crates/engine/src/cmd/sheet.rs index 24e626828..bee10b113 100644 --- a/crates/engine/src/cmd/sheet.rs +++ b/crates/engine/src/cmd/sheet.rs @@ -563,6 +563,7 @@ fn move_sheet(s: &mut Session, p: &Json) -> Result { let name = (2..).map(|k| format!("{base} ({k})")).find(|nm| cx.wb.sheet_index(nm).is_none()).unwrap_or(base.clone()); sh.name = name.chars().take(31).collect(); sh.tables.clear(); // table names must be unique + sh.preserved = None; // kept file content (form controls, OLE objects…) stays with the original let at = to.min(n); // The copy gets its own copies of the sheet's names, referring to the copy. let copies: Vec = cx diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index e3b8b6386..7dd43dfac 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -8,6 +8,7 @@ mod b64; pub mod cell; mod cellmap; pub mod features; +pub mod preserved; pub mod sheet; pub mod store; pub mod style; @@ -15,6 +16,7 @@ pub mod workbook; pub use cell::{Cell, CellPicture, Formula}; pub use features::*; +pub use preserved::{Preserved, PreservedPart, PreservedRel, PreservedXml}; pub use sheet::{DEFAULT_COL_WIDTH, DEFAULT_ROW_HEIGHT, LineIndex, Sheet, Visibility}; pub use store::CellStore; pub use style::{ diff --git a/crates/model/src/preserved.rs b/crates/model/src/preserved.rs new file mode 100644 index 000000000..6d0b35d5c --- /dev/null +++ b/crates/model/src/preserved.rs @@ -0,0 +1,70 @@ +//! File content GridCraft does not model, carried through unchanged so that saving a workbook +//! never silently drops it (XLSX pass-through: custom XML, external links, connections, query +//! tables, form controls, OLE objects…). +//! +//! The data is opaque to everything except the file-format crate: the model only holds it, so a +//! sheet's kept content disappears with the sheet. It is not part of the JSON format. + +use std::sync::Arc; + +/// A relationship from an owning part (the package root, the workbook or a sheet, or a kept part) +/// to a kept part or to an external target. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PreservedRel { + /// The id in the source file (`rId7`). Ids of an owner GridCraft writes are reassigned on save. + pub id: String, + /// The full relationship type URI. + pub rel_type: String, + /// The original part name of the target (no leading `/`), or the raw target of an external + /// relationship. External targets are never opened. + pub target: String, + pub external: bool, +} + +/// A kept package part. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PreservedPart { + /// Part name in the source file (no leading `/`). + pub name: String, + pub content_type: String, + pub data: Arc<[u8]>, + /// The part's own relationships, ids unchanged (the part's bytes refer to them). + pub rels: Vec, +} + +/// A raw XML element of an owning part that refers to kept parts, re-emitted on save with its +/// relationship ids rewritten. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PreservedXml { + /// Which slot of the owner it belongs in (`externalReferences`, `oleObjects`, `controls`). + pub slot: String, + /// The element, namespace declarations it needs included. + pub xml: String, +} + +/// Unmodelled content kept for one owner (the workbook, which also holds package-root content, +/// or one sheet). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Preserved { + /// Relationships of the package root (`_rels/.rels`) to kept parts (workbook owner only). + pub package_rels: Vec, + /// Relationships of the owning workbook or sheet part to kept parts. + pub rels: Vec, + /// Every kept part reachable from `package_rels` and `rels`. + pub parts: Vec, + /// Owner XML elements that refer to kept parts. + pub elements: Vec, + /// The sheet's legacy VML drawing when kept elements need its shapes (form controls, OLE + /// objects). Its own relationships point into `parts`. + pub vml: Option, +} + +impl Preserved { + pub fn is_empty(&self) -> bool { + self.package_rels.is_empty() && self.rels.is_empty() && self.parts.is_empty() && self.elements.is_empty() && self.vml.is_none() + } + /// Total bytes held. + pub fn byte_len(&self) -> usize { + self.parts.iter().chain(self.vml.iter()).fold(0usize, |n, p| n.saturating_add(p.data.len())) + } +} diff --git a/crates/model/src/sheet.rs b/crates/model/src/sheet.rs index d2d82ed94..79fd3fdf3 100644 --- a/crates/model/src/sheet.rs +++ b/crates/model/src/sheet.rs @@ -68,6 +68,10 @@ pub struct Sheet { /// Last selection and scroll position, saved with the file. pub view_active: CellRef, pub view_top_left: CellRef, + /// Unmodelled file content of this sheet kept for saving (form controls, OLE objects, query + /// tables). Goes away with the sheet; not in JSON. + #[serde(skip)] + pub preserved: Option>, } /// Default width of a column (points) and height of a row (points) for the default font. @@ -111,6 +115,7 @@ impl Sheet { print: PrintSettings::default(), view_active: CellRef::default(), view_top_left: CellRef::default(), + preserved: None, } } diff --git a/crates/model/src/workbook.rs b/crates/model/src/workbook.rs index 6ccef9e9c..7e03cb31b 100644 --- a/crates/model/src/workbook.rs +++ b/crates/model/src/workbook.rs @@ -71,6 +71,9 @@ pub struct Workbook { pub custom_lists: Vec>, /// Named cell styles (Cell Styles gallery): name → style. pub cell_styles: Vec<(String, crate::style::Style)>, + /// Unmodelled file content kept for saving (package root and workbook level). Not in JSON. + #[serde(skip)] + pub preserved: Option>, } impl Default for Workbook { @@ -94,6 +97,7 @@ impl Workbook { protected_structure: false, custom_lists: vec![], cell_styles: vec![], + preserved: None, } } pub fn sheet(&self, i: usize) -> Option<&Sheet> { diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 05d1afc77..e5d1bc42a 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -19,6 +19,7 @@ mod drawing; mod fmla; mod ods; mod package; +mod passthrough; mod pivot; mod read; mod richdata; diff --git a/crates/xlsx/src/passthrough.rs b/crates/xlsx/src/passthrough.rs new file mode 100644 index 000000000..187009bca --- /dev/null +++ b/crates/xlsx/src/passthrough.rs @@ -0,0 +1,898 @@ +//! Pass-through of package content GridCraft does not model, so that open → edit → save keeps it. +//! +//! On read, the raw bytes of unmodelled parts are kept with their content type and their +//! relationship graph, per owner: the package root (custom document properties, ribbon +//! customisation, add-in task panes), the workbook (custom XML, links to external workbooks with +//! their ``, data connections) and each sheet (query tables, form controls, +//! ActiveX controls, OLE objects, the sheet background and printer settings, with the sheet's +//! ``, `` and `` XML, the `r:id` of `` and the legacy +//! VML shapes those controls need). +//! +//! On write, kept parts get fresh part names where needed and new relationship ids; the `r:id`s in +//! kept XML are rewritten to match. Kept content is conservative: +//! - Anything whose relationship graph reaches a part GridCraft reads and rewrites (worksheets, +//! drawings, charts, tables, PivotTables, comments, slicers…) is not kept: it would point at +//! content that no longer exists in the same form. +//! - External relationship targets (the path of a linked workbook) are kept as text and never +//! opened, followed or refreshed. +//! - Sheet content lives on the sheet: deleting the sheet drops it, Copy Sheet doesn't copy it. +//! Cell references inside kept parts (a control's linked cell, a query table's range) are kept +//! as they were, not adjusted to inserted or deleted rows. +//! - Strict Open XML files are not passed through (their namespaces differ from what we write). +//! - Sizes are bounded: per part and per workbook. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::fmt::Write as _; +use std::sync::Arc; + +use gridcraft_model::{Preserved, PreservedPart, PreservedRel, PreservedXml, Sheet, Workbook}; +use quick_xml::events::attributes::Attribute; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::name::QName; +use quick_xml::{Reader, Writer}; + +use crate::package::{self, normalize}; +use crate::read::Ctx; +use crate::write::{NS_REL, Out, Rels, XML_DECL}; +use crate::xml::{self, esc_attr}; + +/// Most bytes kept for one workbook. +pub const MAX_KEPT_TOTAL: usize = 256 * 1024 * 1024; +/// Largest single kept part. +pub const MAX_KEPT_PART: u64 = 64 * 1024 * 1024; +/// Most parts kept for one workbook. +const MAX_KEPT_PARTS: usize = 4096; +/// Most relationships read from one rels part. +const MAX_RELS: usize = 10_000; +/// Largest relationships, content-types or VML part read here. +const MAX_META: u64 = 16 * 1024 * 1024; +/// Longest chain of relationships followed from an owner. +const MAX_CHAIN: usize = 16; +/// Largest kept XML fragment of a sheet. +const MAX_FRAGMENT: usize = 4 * 1024 * 1024; +/// Deepest element nesting accepted in a kept fragment. +const MAX_FRAGMENT_DEPTH: usize = 256; + +const NS_REL_STRICT: &str = "http://purl.oclc.org/ooxml/officeDocument/relationships"; +const CT_VML: &str = "application/vnd.openxmlformats-officedocument.vmlDrawing"; + +/// Relationship kinds (last segment of the type) kept from the package root. +const PACKAGE_ROOTS: &[&str] = &["custom-properties", "extensibility", "webextensiontaskpanes"]; +/// Relationship kinds kept from the workbook without a workbook element. +const WORKBOOK_ROOTS: &[&str] = &["customXml", "connections"]; +/// Relationship kinds kept from a sheet without a sheet element. +const SHEET_ROOTS: &[&str] = &["queryTable"]; +/// Sheet relationship kinds that mean the sheet XML may hold elements worth keeping. +const SHEET_ELEMENT_KINDS: &[&str] = &["oleObject", "ctrlProp", "control", "image", "package", "printerSettings"]; +/// Kept sheet elements, in schema order (all after ``, before ``). +const SHEET_SLOTS: &[&str] = &["picture", "oleObjects", "controls"]; + +/// Relationship kinds GridCraft reads and rewrites, or that depend on content it rewrites: +/// kept content whose graph reaches one of them is dropped. +const REWRITTEN: &[&str] = &[ + "officedocument", + "core-properties", + "extended-properties", + "worksheet", + "chartsheet", + "dialogsheet", + "xlmacrosheet", + "drawing", + "chart", + "chartex", + "chartstyle", + "chartcolorstyle", + "chartusershapes", + "table", + "pivottable", + "pivotcachedefinition", + "pivotcacherecords", + "comments", + "vmldrawing", + "threadedcomment", + "person", + "sharedstrings", + "styles", + "theme", + "sheetmetadata", + "calcchain", + "vbaproject", + "rdrichvalue", + "rdrichvaluestructure", + "rdrichvaluetypes", + "rdrichvaluewebimage", + "rdarray", + "rdsupportingpropertybag", + "rdsupportingpropertybagstructure", + "richvaluerel", + "slicer", + "slicercache", + "timeline", + "timelinecache", + "powerpivotdata", +]; + +/// Part-name prefixes (lower case) GridCraft's writer never produces: kept parts there keep their +/// original names. Anything else is renamed so it can't collide with written parts. +const OWN_DIRS: &[&str] = &[ + "customxml/", + "docprops/custom.xml", + "customui/", + "xl/externallinks/", + "xl/connections.xml", + "xl/querytables/", + "xl/ctrlprops/", + "xl/embeddings/", + "xl/activex/", + "xl/webextensions/", + "xl/printersettings/", +]; + +fn kind(rel_type: &str) -> &str { + rel_type.rsplit('/').next().unwrap_or("") +} + +fn is_kind(r: &PreservedRel, kinds: &[&str]) -> bool { + let k = kind(&r.rel_type); + kinds.iter().any(|x| x.eq_ignore_ascii_case(k)) +} + +fn rewritten(r: &PreservedRel) -> bool { + !r.external && REWRITTEN.contains(&kind(&r.rel_type).to_ascii_lowercase().as_str()) +} + +fn rels_part_name(part: &str) -> String { + let (dir, file) = package::split_dir(part); + if dir.is_empty() { format!("_rels/{file}.rels") } else { format!("{dir}/_rels/{file}.rels") } +} + +/// A part's relationships with full type URIs. `Ok(vec![])` when it has none; an error when the +/// rels part is damaged or too large (its part then can't be kept faithfully). +fn raw_rels(cx: &mut Ctx<'_>, part: &str) -> Result, String> { + let name = rels_part_name(part); + let bytes = match cx.pkg.read_limited(&name, MAX_META) { + Ok(Some(b)) => b, + Ok(None) => return Ok(vec![]), + Err(e) => return Err(e.to_string()), + }; + let root = xml::parse(&bytes).map_err(|e| format!("{name}: {e}"))?; + let (dir, _) = package::split_dir(part); + let mut out = vec![]; + for r in root.kids("Relationship") { + if out.len() >= MAX_RELS { + return Err(format!("{name}: too many relationships")); + } + let (Some(id), Some(target)) = (r.attr("Id"), r.attr("Target")) else { continue }; + let external = r.attr("TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External")); + out.push(PreservedRel { + id: id.to_string(), + rel_type: r.attr("Type").unwrap_or("").to_string(), + target: if external { target.to_string() } else { package::resolve(dir, target) }, + external, + }); + } + Ok(out) +} + +// ---------------------------------------------------------------- reading + +/// Collects kept content while a package is read. +pub struct KeptReader { + on: bool, + strict: bool, + defaults: HashMap, + overrides: HashMap, + budget: usize, + parts_left: usize, +} + +impl KeptReader { + pub fn new(cx: &mut Ctx<'_>) -> KeptReader { + let mut k = KeptReader { + on: true, + strict: false, + defaults: HashMap::new(), + overrides: HashMap::new(), + budget: MAX_KEPT_TOTAL, + parts_left: MAX_KEPT_PARTS, + }; + if let Ok(root) = raw_rels(cx, "") + && root.iter().any(|r| r.rel_type.starts_with("http://purl.oclc.org/")) + { + k.on = false; + k.strict = true; + } + match cx.pkg.read_limited("[Content_Types].xml", MAX_META).ok().flatten().and_then(|b| xml::parse(&b).ok()) { + Some(types) => { + for d in types.kids("Default") { + if let (Some(e), Some(t)) = (d.attr("Extension"), d.attr("ContentType")) { + k.defaults.insert(e.to_ascii_lowercase(), t.to_string()); + } + } + for o in types.kids("Override") { + if let (Some(p), Some(t)) = (o.attr("PartName"), o.attr("ContentType")) { + k.overrides.insert(normalize(p), t.to_string()); + } + } + } + None => k.on = false, + } + k + } + + fn content_type(&self, part: &str) -> Option { + let key = normalize(part); + if let Some(t) = self.overrides.get(&key) { + return Some(t.clone()); + } + let (_, file) = package::split_dir(&key); + let ext = file.rsplit_once('.').map(|(_, e)| e)?; + self.defaults.get(ext).cloned() + } + + /// The parts reachable from `root` that aren't in `seen` yet. Fails (and keeps nothing) when the + /// graph reaches rewritten content, a missing or damaged part, or a size limit. + fn closure(&mut self, cx: &mut Ctx<'_>, root: &PreservedRel, seen: &HashSet) -> Result, String> { + if root.external { + return Ok(vec![]); + } + if rewritten(root) { + return Err(format!("it refers to {}", kind(&root.rel_type))); + } + let mut parts = vec![]; + let mut bytes = 0usize; + let mut local: HashSet = HashSet::new(); + let mut queue = VecDeque::from([(root.target.clone(), 0usize)]); + while let Some((name, depth)) = queue.pop_front() { + let key = normalize(&name); + if seen.contains(&key) || !local.insert(key) { + continue; + } + if depth > MAX_CHAIN { + return Err("its relationships are nested too deeply".into()); + } + if parts.len() >= self.parts_left { + return Err("it has too many parts".into()); + } + let content_type = self.content_type(&name).ok_or_else(|| format!("{name} has no content type"))?; + let data = match cx.pkg.read_limited(&name, MAX_KEPT_PART) { + Ok(Some(d)) => d, + Ok(None) => return Err(format!("{name} is missing")), + Err(e) => return Err(e.to_string()), + }; + bytes = bytes.saturating_add(data.len()); + if bytes > self.budget { + return Err("it is too large".into()); + } + let rels = raw_rels(cx, &name)?; + for r in rels.iter().filter(|r| !r.external) { + if rewritten(r) { + return Err(format!("it refers to {}", kind(&r.rel_type))); + } + queue.push_back((r.target.clone(), depth + 1)); + } + parts.push(PreservedPart { name, content_type, data: Arc::from(data), rels }); + } + self.budget = self.budget.saturating_sub(bytes); + self.parts_left = self.parts_left.saturating_sub(parts.len()); + Ok(parts) + } + + /// Keeps a set of relationships (all or nothing) and the parts behind them. + fn keep(&mut self, cx: &mut Ctx<'_>, p: &mut Preserved, roots: &[PreservedRel], seen: &mut HashSet) -> Result<(), String> { + let mut new_parts: Vec = vec![]; + let mut scratch = seen.clone(); + for r in roots { + let parts = self.closure(cx, r, &scratch)?; + scratch.extend(parts.iter().map(|q| normalize(&q.name))); + new_parts.extend(parts); + } + *seen = scratch; + p.parts.extend(new_parts); + for r in roots { + if !p.rels.iter().any(|x| x.id == r.id) { + p.rels.push(r.clone()); + } + } + Ok(()) + } + + /// Package-root and workbook-level content. + pub fn workbook(&mut self, cx: &mut Ctx<'_>, wb_part: &str, wb_xml: &xml::El) -> Option> { + if !self.on { + if self.strict { + cx.warn("unmodelled content of a Strict Open XML workbook (custom XML, links, controls…) is not kept on save"); + } + return None; + } + let mut p = Preserved::default(); + let mut seen = HashSet::new(); + // Package root: custom document properties, ribbon customisation, task panes. + if let Ok(root) = raw_rels(cx, "") { + for r in root.iter().filter(|r| is_kind(r, PACKAGE_ROOTS)) { + let mut q = Preserved::default(); + match self.keep(cx, &mut q, std::slice::from_ref(r), &mut seen) { + Ok(()) => { + p.parts.extend(q.parts); + p.package_rels.extend(q.rels); + } + Err(why) => cx.warn(format!("{} was dropped: {why}", kind(&r.rel_type))), + } + } + } + let wb_rels = raw_rels(cx, wb_part).unwrap_or_default(); + for r in wb_rels.iter().filter(|r| is_kind(r, WORKBOOK_ROOTS)) { + if let Err(why) = self.keep(cx, &mut p, std::slice::from_ref(r), &mut seen) { + cx.warn(format!("{} part {} was dropped: {why}", kind(&r.rel_type), r.target)); + } + } + // Links to external workbooks: `[1]` in formulas is the position in , + // so they are kept all together or not at all. + let ids: Vec<&str> = + wb_xml.child("externalReferences").map(|e| e.kids("externalReference").filter_map(|x| x.attr("id")).collect()).unwrap_or_default(); + if !ids.is_empty() { + let roots: Option> = + ids.iter().map(|id| wb_rels.iter().find(|r| r.id == *id && is_kind(r, &["externalLink"])).cloned()).collect(); + let kept = match roots { + Some(roots) => self.keep(cx, &mut p, &roots, &mut seen), + None => Err("a link has no part".into()), + }; + match kept { + Ok(()) => { + let mut x = format!(""); + for id in &ids { + let _ = write!(x, "", esc_attr(id)); + } + x.push_str(""); + p.elements.push(PreservedXml { slot: "externalReferences".into(), xml: x }); + cx.warn("links to external workbooks were kept; they are not followed or refreshed"); + } + Err(why) => cx.warn(format!("links to external workbooks were dropped: {why}")), + } + } + (!p.is_empty()).then(|| Arc::new(p)) + } + + /// Sheet-level content of the sheet part `part`. + pub fn sheet(&mut self, cx: &mut Ctx<'_>, part: &str, name: &str) -> Option> { + if !self.on { + return None; + } + let rels = match raw_rels(cx, part) { + Ok(r) => r, + Err(e) => { + cx.warn(format!("sheet \"{name}\": relationships could not be read ({e}); unmodelled content was dropped")); + return None; + } + }; + let mut p = Preserved::default(); + let mut seen = HashSet::new(); + for r in rels.iter().filter(|r| is_kind(r, SHEET_ROOTS)) { + if let Err(why) = self.keep(cx, &mut p, std::slice::from_ref(r), &mut seen) { + cx.warn(format!("sheet \"{name}\": {} was dropped: {why}", kind(&r.rel_type))); + } + } + if rels.iter().any(|r| is_kind(r, SHEET_ELEMENT_KINDS)) { + self.sheet_elements(cx, part, name, &rels, &mut p, &mut seen); + } + (!p.is_empty()).then(|| Arc::new(p)) + } + + fn sheet_elements(&mut self, cx: &mut Ctx<'_>, part: &str, name: &str, rels: &[PreservedRel], p: &mut Preserved, seen: &mut HashSet) { + let scan = match cx.pkg.read(part) { + Ok(Some(bytes)) => scan_sheet(&bytes), + _ => None, + }; + let Some(scan) = scan else { + cx.warn(format!("sheet \"{name}\": controls and embedded objects could not be read and were dropped")); + return; + }; + // Printer settings (a binary part per sheet) hang off . + if let Some(id) = scan.page_setup.as_deref() + && let Some(r) = rels.iter().find(|r| r.id == id && is_kind(r, &["printerSettings"])) + { + match self.keep(cx, p, std::slice::from_ref(r), seen) { + Ok(()) => p.elements.push(PreservedXml { slot: "pageSetup".into(), xml: r.id.clone() }), + Err(why) => cx.warn(format!("sheet \"{name}\": printer settings were dropped: {why}")), + } + } + let mut frags = scan.fragments; + // Controls and OLE objects are drawn from shapes in the legacy VML drawing. + if frags.iter().any(|(slot, _)| slot != "picture") { + match self.vml(cx, rels, scan.legacy_drawing.as_deref(), seen) { + Ok((vml, parts)) => { + p.parts.extend(parts); + p.vml = Some(vml); + } + Err(why) => { + cx.warn(format!("sheet \"{name}\": controls and embedded objects were dropped: {why}")); + frags.retain(|(slot, _)| slot == "picture"); + } + } + } + for (slot, x) in frags { + let mut ids = vec![]; + let ok = map_rel_ids(&x, &mut |id| { + ids.push(id.to_string()); + Some(id.to_string()) + }) + .is_some(); + let roots: Option> = ids.iter().map(|id| rels.iter().find(|r| r.id == *id).cloned()).collect(); + let kept = match (ok, roots) { + (true, Some(roots)) => self.keep(cx, p, &roots, seen), + (false, _) => Err("its XML is damaged".into()), + (_, None) => Err("it refers to a missing relationship".into()), + }; + match kept { + Ok(()) => p.elements.push(PreservedXml { slot, xml: x }), + Err(why) => cx.warn(format!("sheet \"{name}\": {slot} were dropped: {why}")), + } + } + if p.elements.iter().any(|e| e.slot == "oleObjects" || e.slot == "controls") { + cx.warn(format!("sheet \"{name}\": embedded objects and form controls are not shown; they are kept for saving")); + } else { + p.vml = None; + } + } + + /// The sheet's VML drawing without its notes (GridCraft writes those), and the parts its + /// shapes refer to (preview images). + fn vml( + &mut self, + cx: &mut Ctx<'_>, + rels: &[PreservedRel], + rid: Option<&str>, + seen: &mut HashSet, + ) -> Result<(PreservedPart, Vec), String> { + let rel = rid.and_then(|id| rels.iter().find(|r| r.id == id && !r.external)).ok_or("the sheet has no VML drawing")?; + let data = match cx.pkg.read_limited(&rel.target, MAX_META) { + Ok(Some(d)) => d, + Ok(None) => return Err(format!("{} is missing", rel.target)), + Err(e) => return Err(e.to_string()), + }; + let text = String::from_utf8(data).map_err(|_| "the VML drawing is not UTF-8".to_string())?; + let text = strip_notes(&text); + let vrels = raw_rels(cx, &rel.target)?; + let mut q = Preserved::default(); + let roots: Vec = vrels.iter().filter(|r| !r.external).cloned().collect(); + self.keep(cx, &mut q, &roots, seen)?; + let vml = PreservedPart { name: rel.target.clone(), content_type: CT_VML.into(), data: Arc::from(text.into_bytes()), rels: vrels }; + Ok((vml, q.parts)) + } +} + +/// What a sheet part holds for pass-through. +struct SheetScan { + /// (slot, XML with the namespace declarations it needs). + fragments: Vec<(String, String)>, + legacy_drawing: Option, + /// The relationship id on `` (printer settings). + page_setup: Option, +} + +/// Finds the sheet's ``, ``, `` (also inside +/// ``) and the `` id. `None` when the XML is damaged. +fn scan_sheet(bytes: &[u8]) -> Option { + let data = xml::to_utf8(bytes); + let text = std::str::from_utf8(&data).ok()?; + let mut reader = Reader::from_str(text); + let mut decls: Vec<(String, String)> = vec![]; + let mut out = SheetScan { fragments: vec![], legacy_drawing: None, page_setup: None }; + let mut in_root = false; + loop { + let start = usize::try_from(reader.buffer_position()).ok()?; + match reader.read_event().ok()? { + Event::Eof => break, + Event::Start(e) if !in_root => { + in_root = true; + for a in e.attributes().with_checks(false).flatten() { + if let Some(p) = a.key.as_ref().strip_prefix(b"xmlns:") + && let (Ok(p), Ok(v)) = (std::str::from_utf8(p), std::str::from_utf8(&a.value)) + && !v.contains('"') + { + decls.push((p.to_string(), v.to_string())); + } + } + } + Event::Start(e) => { + let local = e.local_name().as_ref().to_vec(); + match local.as_slice() { + b"pageSetup" => out.page_setup = rel_id_attr(&e), + b"legacyDrawing" => out.legacy_drawing = rel_id_attr(&e), + _ => {} + } + reader.read_to_end(e.name()).ok()?; + let end = usize::try_from(reader.buffer_position()).ok()?; + let wanted = matches!(local.as_slice(), b"picture" | b"oleObjects" | b"controls" | b"AlternateContent"); + if wanted && end.saturating_sub(start) <= MAX_FRAGMENT { + let frag = text.get(start..end)?; + push_fragment(&mut out, frag, &local, &decls); + } + } + Event::Empty(e) if in_root => { + let local = e.local_name().as_ref().to_vec(); + if local == b"legacyDrawing" || local == b"pageSetup" { + let id = rel_id_attr(&e); + if local == b"pageSetup" { + out.page_setup = id; + } else { + out.legacy_drawing = id; + } + } else if local == b"picture" + && let Some(frag) = text.get(start..usize::try_from(reader.buffer_position()).ok()?) + { + push_fragment(&mut out, frag, &local, &decls); + } + } + Event::End(_) => break, + _ => {} + } + } + Some(out) +} + +/// The `r:id` of an element (an `id` attribute with a namespace prefix). +fn rel_id_attr(e: &BytesStart<'_>) -> Option { + e.attributes() + .with_checks(false) + .flatten() + .find(|a| a.key.local_name().as_ref() == b"id" && a.key.prefix().is_some()) + .and_then(|a| String::from_utf8(a.value.to_vec()).ok()) +} + +fn push_fragment(out: &mut SheetScan, frag: &str, local: &[u8], decls: &[(String, String)]) { + let frag = with_decls(frag, decls); + let slot = if local == b"AlternateContent" { + // The kept element is the one the first choice holds. + let Ok(el) = xml::parse(frag.as_bytes()) else { return }; + match el.child("Choice").and_then(|c| c.children.first()) { + Some(c) => c.name.clone(), + None => return, + } + } else { + String::from_utf8_lossy(local).into_owned() + }; + if SHEET_SLOTS.contains(&slot.as_str()) && !out.fragments.iter().any(|(s, _)| *s == slot) { + out.fragments.push((slot, frag)); + } +} + +/// Adds the root's namespace declarations a fragment doesn't declare itself. +fn with_decls(frag: &str, decls: &[(String, String)]) -> String { + let Some(name_end) = frag.char_indices().skip(1).find(|(_, c)| c.is_whitespace() || *c == '/' || *c == '>').map(|(i, _)| i) else { + return frag.to_string(); + }; + let head = frag.get(..frag.find('>').unwrap_or(frag.len())).unwrap_or(""); + let mut add = String::new(); + for (p, v) in decls { + if !head.contains(&format!("xmlns:{p}=")) { + let _ = write!(add, " xmlns:{p}=\"{v}\""); + } + } + format!("{}{add}{}", frag.get(..name_end).unwrap_or(""), frag.get(name_end..).unwrap_or("")) +} + +/// Byte ranges of the `…` (or self-closed) elements in VML text. +fn shape_spans(vml: &str) -> Vec<(usize, usize)> { + let mut spans = vec![]; + let mut from = 0; + while let Some(i) = vml.get(from..).and_then(|s| s.find("' || c == '/'); + if !is_shape { + from = after; + continue; + } + let Some(tag_end) = vml.get(i..).and_then(|s| s.find('>')).map(|j| i + j + 1) else { break }; + let end = if vml.get(..tag_end).is_some_and(|t| t.ends_with("/>")) { + tag_end + } else { + match vml.get(tag_end..).and_then(|s| s.find("")) { + Some(j) => tag_end + j + "".len(), + None => break, + } + }; + spans.push((i, end)); + from = end; + } + spans +} + +/// VML without its note shapes (`ObjectType="Note"`). +fn strip_notes(vml: &str) -> String { + let mut out = String::with_capacity(vml.len()); + let mut last = 0; + for (a, b) in shape_spans(vml) { + let shape = vml.get(a..b).unwrap_or(""); + if shape.contains("ObjectType=\"Note\"") { + out.push_str(vml.get(last..a).unwrap_or("")); + last = b; + } + } + out.push_str(vml.get(last..).unwrap_or("")); + out +} + +/// Shape numbers (`_x0000_s1025` → 1025) used in VML text. +fn shape_numbers(vml: &str) -> Vec { + vml.match_indices("_x0000_s") + .filter_map(|(i, m)| { + let digits: String = vml.get(i + m.len()..)?.chars().take_while(char::is_ascii_digit).take(18).collect(); + digits.parse().ok() + }) + .collect() +} + +/// Kept VML (controls, OLE objects) plus GridCraft's note shapes: the notes get ids in a fresh +/// shape block so they can't collide with the kept shapes the sheet XML refers to. +fn merge_vml(kept: &str, ours: &str) -> String { + let Some(close) = kept.rfind("") else { return ours.to_string() }; + let block = shape_numbers(kept).into_iter().max().unwrap_or(0) / 1024 + 1; + let mut add = String::new(); + if !kept.contains("id=\"_x0000_t202\"") + && let Some(i) = ours.find("")) + { + add.push_str(ours.get(i..i + j + "".len()).unwrap_or("")); + } + for (k, (a, b)) in shape_spans(ours).into_iter().enumerate() { + let shape = ours.get(a..b).unwrap_or(""); + let id = block.saturating_mul(1024).saturating_add(1 + k as u64); + match (shape.find("id=\"_x0000_s"), shape.find("id=\"_x0000_s").and_then(|i| shape.get(i + 4..)?.find('"').map(|j| i + 4 + j))) { + (Some(i), Some(j)) => { + let _ = write!(add, "{}id=\"_x0000_s{id}\"{}", shape.get(..i).unwrap_or(""), shape.get(j + 1..).unwrap_or("")); + } + _ => add.push_str(shape), + } + } + let mut head = kept.get(..close).unwrap_or("").to_string(); + // Claim the block in the drawing's id map. + if let Some(i) = head.find(" Option) -> Option { + let mut reader = Reader::from_str(x); + let mut writer = Writer::new(Vec::with_capacity(x.len())); + let mut prefixes: HashSet> = HashSet::new(); + let mut depth = 0usize; + loop { + match reader.read_event().ok()? { + Event::Eof => break, + Event::Start(e) => { + depth += 1; + if depth > MAX_FRAGMENT_DEPTH { + return None; + } + let e = map_start(&e, &mut prefixes, map)?; + writer.write_event(Event::Start(e)).ok()?; + } + Event::Empty(e) => { + let e = map_start(&e, &mut prefixes, map)?; + writer.write_event(Event::Empty(e)).ok()?; + } + Event::End(e) => { + depth = depth.saturating_sub(1); + writer.write_event(Event::End(e)).ok()?; + } + ev => writer.write_event(ev).ok()?, + } + } + String::from_utf8(writer.into_inner()).ok() +} + +fn map_start(e: &BytesStart<'_>, prefixes: &mut HashSet>, map: &mut dyn FnMut(&str) -> Option) -> Option> { + for a in e.attributes().with_checks(false) { + let a = a.ok()?; + if let Some(p) = a.key.as_ref().strip_prefix(b"xmlns:") + && (a.value.as_ref() == NS_REL.as_bytes() || a.value.as_ref() == NS_REL_STRICT.as_bytes()) + { + prefixes.insert(p.to_vec()); + } + } + let name = String::from_utf8(e.name().as_ref().to_vec()).ok()?; + let mut out = BytesStart::new(name); + for a in e.attributes().with_checks(false) { + let a = a.ok()?; + let key = a.key.as_ref(); + let is_rel = !key.starts_with(b"xmlns") && a.key.prefix().is_some_and(|p| prefixes.contains(p.as_ref())); + if is_rel { + let old = std::str::from_utf8(&a.value).ok()?; + let new = map(old)?; + out.push_attribute(Attribute { key: QName(key), value: Cow::Owned(esc_attr(&new).into_bytes()) }); + } else { + out.push_attribute(Attribute { key: QName(key), value: a.value.clone() }); + } + } + Some(out) +} + +// ---------------------------------------------------------------- writing + +/// Names given to kept parts while a package is written. +#[derive(Default)] +pub struct KeptOut { + used: HashSet, + n: u32, + /// Package-root relationships (type, target, external), added when `_rels/.rels` is written. + root: Vec<(String, String, bool)>, +} + +impl KeptOut { + /// A part name for a kept part: its original name when that can't collide with anything + /// written, otherwise a fresh one in the same folder. + fn claim(&mut self, original: &str) -> String { + let lower = original.to_ascii_lowercase(); + let own = OWN_DIRS.iter().any(|d| lower.starts_with(d)); + if own && self.used.insert(lower) { + return original.to_string(); + } + let (dir, file) = package::split_dir(original); + let (stem, ext) = match file.rsplit_once('.') { + Some((s, e)) => (s, format!(".{e}")), + None => (file, String::new()), + }; + let stem = if own { format!("{stem}_") } else { "kept".to_string() }; + loop { + self.n = self.n.saturating_add(1); + let file = format!("{stem}{}{ext}", self.n); + let cand = if dir.is_empty() { file } else { format!("{dir}/{file}") }; + if self.used.insert(cand.to_ascii_lowercase()) || self.n == u32::MAX { + return cand; + } + } + } +} + +/// Original part name (normalized) → written part name. +type NameMap = HashMap; + +fn rel_target(r: &PreservedRel, names: &NameMap) -> Option { + if r.external { Some(r.target.clone()) } else { names.get(&normalize(&r.target)).map(|n| format!("/{n}")) } +} + +/// A kept part's relationships, ids unchanged, targets renamed. +fn part_rels_xml(rels: &[PreservedRel], names: &NameMap) -> String { + let mut s = format!("{XML_DECL}"); + for r in rels { + let Some(target) = rel_target(r, names) else { continue }; + let _ = write!( + s, + "", + esc_attr(&r.id), + esc_attr(&r.rel_type), + esc_attr(&target), + if r.external { " TargetMode=\"External\"" } else { "" } + ); + } + s.push_str(""); + s +} + +fn write_parts(out: &mut Out, parts: &[PreservedPart]) -> NameMap { + let mut names = NameMap::new(); + for p in parts { + names.entry(normalize(&p.name)).or_insert_with(|| out.kept.claim(&p.name)); + } + let mut written = HashSet::new(); + for p in parts { + let Some(new) = names.get(&normalize(&p.name)).cloned() else { continue }; + if !written.insert(new.clone()) { + continue; + } + out.part(&new, Some(&p.content_type), p.data.to_vec()); + if !p.rels.is_empty() { + out.part(&rels_part_name(&new), None, part_rels_xml(&p.rels, &names).into_bytes()); + } + } + names +} + +/// Adds the owner's relationships; returns old id → new id. +fn attach(rels: &mut Rels, list: &[PreservedRel], names: &NameMap) -> HashMap { + let mut ids = HashMap::new(); + for r in list { + if let Some(target) = rel_target(r, names) { + let id = rels.add_raw(&r.rel_type, &target, r.external); + ids.insert(r.id.clone(), id); + } + } + ids +} + +fn remap(x: &str, ids: &HashMap) -> Option { + map_rel_ids(x, &mut |id| ids.get(id).cloned()) +} + +/// Writes the workbook's kept parts and relationships; returns the workbook XML elements that +/// go right after `` (``). +pub fn write_workbook(wb: &Workbook, wb_rels: &mut Rels, out: &mut Out) -> String { + let Some(p) = wb.preserved.as_deref() else { return String::new() }; + let names = write_parts(out, &p.parts); + let ids = attach(wb_rels, &p.rels, &names); + for r in &p.package_rels { + if let Some(target) = rel_target(r, &names) { + out.kept.root.push((r.rel_type.clone(), target, r.external)); + } + } + p.elements.iter().filter(|e| e.slot == "externalReferences").filter_map(|e| remap(&e.xml, &ids)).collect() +} + +/// Adds the kept package-root relationships (custom properties…) to `_rels/.rels`. +pub fn write_package_rels(root: &mut Rels, out: &mut Out) { + for (ty, target, external) in std::mem::take(&mut out.kept.root) { + root.add_raw(&ty, &target, external); + } +} + +/// A sheet's kept content being written. +pub struct SheetKept<'a> { + p: Option<&'a Preserved>, + names: NameMap, + ids: HashMap, +} + +impl<'a> SheetKept<'a> { + /// Writes the sheet's kept parts and adds their relationships to the sheet's. + pub fn write(sheet: &'a Sheet, rels: &mut Rels, out: &mut Out) -> SheetKept<'a> { + let Some(p) = sheet.preserved.as_deref() else { return SheetKept { p: None, names: NameMap::new(), ids: HashMap::new() } }; + let names = write_parts(out, &p.parts); + let ids = attach(rels, &p.rels, &names); + SheetKept { p: Some(p), names, ids } + } + + fn vml(&self) -> Option<&'a PreservedPart> { + self.p?.vml.as_ref() + } + + fn write_vml_rels(&self, vml: &PreservedPart, name: &str, out: &mut Out) { + if !vml.rels.is_empty() { + out.part(&rels_part_name(name), None, part_rels_xml(&vml.rels, &self.names).into_bytes()); + } + } + + /// GridCraft's notes VML for part `name`, merged with the kept control and object shapes. + pub fn notes_vml(&self, ours: String, name: &str, out: &mut Out) -> String { + let Some(vml) = self.vml() else { return ours }; + self.write_vml_rels(vml, name, out); + merge_vml(&String::from_utf8_lossy(&vml.data), &ours) + } + + /// The kept VML drawing of a sheet without notes; its new relationship id. + pub fn lone_vml(&self, rels: &mut Rels, out: &mut Out) -> Option { + let vml = self.vml()?; + let name = out.kept.claim(&vml.name); + out.part(&name, Some(CT_VML), vml.data.to_vec()); + self.write_vml_rels(vml, &name, out); + Some(rels.add("vmlDrawing", &format!("/{name}"))) + } + + /// The new relationship id for `` (printer settings). + pub fn page_setup_id(&self) -> Option<&str> { + let old = &self.p?.elements.iter().find(|e| e.slot == "pageSetup")?.xml; + self.ids.get(old).map(String::as_str) + } + + /// The kept sheet elements, in schema order. + pub fn elements(&self) -> String { + let Some(p) = self.p else { return String::new() }; + let mut s = String::new(); + for slot in SHEET_SLOTS { + for e in p.elements.iter().filter(|e| e.slot == *slot) { + if let Some(x) = remap(&e.xml, &self.ids) { + s.push_str(&x); + } + } + } + s + } +} diff --git a/crates/xlsx/src/read.rs b/crates/xlsx/src/read.rs index 050a9f3c6..50b568e5d 100644 --- a/crates/xlsx/src/read.rs +++ b/crates/xlsx/src/read.rs @@ -98,6 +98,7 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { return Err(IoError::Format(format!("unexpected root element <{}> in workbook part", wb_xml.name))); } let wb_rels = cx.pkg.rels(&wb_part)?; + let mut kept = crate::passthrough::KeptReader::new(&mut cx); cx.cell_pictures = match crate::richdata::read(&mut cx, &wb_rels) { Ok(pictures) => pictures, Err(error) => { @@ -212,6 +213,7 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { Err(IoError::TooLarge(m)) => return Err(IoError::TooLarge(m)), Err(e) => cx.warn(format!("PivotTables on sheet \"{name}\" could not be read: {e}")), } + sheet.preserved = kept.sheet(&mut cx, &rel.target, &name); sheet.visibility = match se.attr("state") { Some("hidden") => Visibility::Hidden, Some("veryHidden") => Visibility::VeryHidden, @@ -308,9 +310,7 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { if wb_rels.iter().any(|r| r.kind == "vbaProject") { cx.warn("macros (VBA project) are not supported and were dropped"); } - if wb_rels.iter().any(|r| r.kind == "externalLink") { - cx.warn("links to external workbooks are not supported"); - } + wb.preserved = kept.workbook(&mut cx, &wb_part, &wb_xml); // Calls of LAMBDA names are spelled like the names (the parser upper-cases them). wb.apply_name_call_case(); wb.styles = cx.styles; diff --git a/crates/xlsx/src/sheet_read.rs b/crates/xlsx/src/sheet_read.rs index 065558e6b..8f51f3320 100644 --- a/crates/xlsx/src/sheet_read.rs +++ b/crates/xlsx/src/sheet_read.rs @@ -183,7 +183,6 @@ pub fn read_sheet(cx: &mut Ctx<'_>, part: &str, name: &str) -> Result read_ext(cx, e, &mut sheet), - "oleObjects" | "controls" => cx.warn(format!("sheet \"{name}\": embedded objects and form controls are not supported")), _ => {} } } diff --git a/crates/xlsx/src/sheet_write.rs b/crates/xlsx/src/sheet_write.rs index b1e3fca2c..cfd1b6bf4 100644 --- a/crates/xlsx/src/sheet_write.rs +++ b/crates/xlsx/src/sheet_write.rs @@ -414,6 +414,8 @@ pub fn write_sheet(wb: &Workbook, si: usize, selected: bool, out: &mut Out) -> ( } s.push_str("/>"); } + // Unmodelled content kept from the source file (printer settings, controls, OLE objects…). + let kept = crate::passthrough::SheetKept::write(sheet, &mut rels, out); let m = |i: usize| num(p.margins.get(i).copied().unwrap_or(0.75) as f64); let _ = write!(s, "", m(0), m(1), m(2), m(3), m(4), m(5)); @@ -429,7 +431,11 @@ pub fn write_sheet(wb: &Workbook, si: usize, selected: bool, out: &mut Out) -> ( if let Some((w, h)) = p.fit_to { let _ = write!(s, " fitToWidth=\"{w}\" fitToHeight=\"{h}\""); } - let _ = write!(s, " orientation=\"{}\"/>", if p.orientation == Orientation::Landscape { "landscape" } else { "portrait" }); + let _ = write!(s, " orientation=\"{}\"", if p.orientation == Orientation::Landscape { "landscape" } else { "portrait" }); + if let Some(id) = kept.page_setup_id() { + let _ = write!(s, " r:id=\"{id}\""); + } + s.push_str("/>"); if !p.header.is_empty() || !p.footer.is_empty() { s.push_str(""); if !p.header.is_empty() { @@ -506,11 +512,16 @@ pub fn write_sheet(wb: &Workbook, si: usize, selected: bool, out: &mut Out) -> ( let n = out.comments; out.part(&format!("xl/comments{n}.xml"), Some(CT_COMMENTS), comments_xml(sheet).into_bytes()); out.default_type("vml", "application/vnd.openxmlformats-officedocument.vmlDrawing"); - out.part(&format!("xl/drawings/vmlDrawing{n}.vml"), None, vml_xml(sheet, n).into_bytes()); + let vml_name = format!("xl/drawings/vmlDrawing{n}.vml"); + let vml = kept.notes_vml(vml_xml(sheet, n), &vml_name, out); + out.part(&vml_name, None, vml.into_bytes()); rels.add("comments", &format!("../comments{n}.xml")); let rid = rels.add("vmlDrawing", &format!("../drawings/vmlDrawing{n}.vml")); let _ = write!(s, ""); + } else if let Some(rid) = kept.lone_vml(&mut rels, out) { + let _ = write!(s, ""); } + s.push_str(&kept.elements()); // Tables. if !tables.is_empty() { let mut parts = String::new(); diff --git a/crates/xlsx/src/tests/mod.rs b/crates/xlsx/src/tests/mod.rs index 43639fee4..cd5dacac3 100644 --- a/crates/xlsx/src/tests/mod.rs +++ b/crates/xlsx/src/tests/mod.rs @@ -5,6 +5,7 @@ mod csv_tests; mod fixtures; mod malformed; mod ods_tests; +mod passthrough_tests; mod pivot_tests; mod richdata_tests; mod roundtrip; diff --git a/crates/xlsx/src/tests/passthrough_tests.rs b/crates/xlsx/src/tests/passthrough_tests.rs new file mode 100644 index 000000000..10b2681d7 --- /dev/null +++ b/crates/xlsx/src/tests/passthrough_tests.rs @@ -0,0 +1,430 @@ +//! Pass-through of unmodelled package content (custom XML, external links, connections, query +//! tables, printer settings, form controls, OLE objects). Packages are built in code. + +use std::collections::HashMap; +use std::io::Read; +use std::sync::Arc; + +use gridcraft_model::Sheet; + +use super::make_zip; +use crate::package::{normalize, resolve, split_dir}; +use crate::xml; +use crate::{read_xlsx, write_xlsx}; + +const NS_R: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const NS_MAIN: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +const PRINTER: &[u8] = &[0, 1, 2, 3, 250, 251, 252, 253, 9, 9, 9]; +const OLE: &[u8] = b"\xD0\xCF\x11\xE0 not really a compound file"; +const EMF: &[u8] = b"\x01\x00\x00\x00 preview"; +const CUSTOM_XML: &str = r#"42"#; +const CUSTOM_PROPS: &str = r#""#; + +fn rels(items: &[(&str, &str, &str)]) -> String { + let mut s = String::from(r#""#); + for (id, ty, target) in items { + let ty = if ty.starts_with("http") { ty.to_string() } else { format!("{NS_R}/{ty}") }; + let ext = if target.contains("://") { r#" TargetMode="External""# } else { "" }; + s.push_str(&format!(r#""#)); + } + s.push_str(""); + s +} + +const CONTENT_TYPES: &str = r#""#; + +const EXTERNAL_LINK: &str = r#"7"#; +const CONNECTIONS: &str = r#""#; +const QUERY_TABLE: &str = r#""#; +const CTRL_PROP: &str = r#""#; +const COMMENTS: &str = r#"Anote"#; +const VML: &str = r##"1, 0, 1, 0, 3, 0, 5, 0Pict4, 0, 1, 0, 6, 0, 2, 0$C$1
old note
00"##; + +/// A workbook with every kind of kept content: package root custom properties, custom XML, an +/// external link (with formulas using it), a connection, and on sheet "Data" a query table, +/// printer settings, an OLE object (in `mc:AlternateContent`), a form control, a note and the VML +/// drawing all of them share. +fn rich_package() -> Vec { + let workbook = format!( + r#""# + ); + let sheet1 = format!( + r#"[1]Sheet1!$A$17SUM([1]Sheet1!A1:A3)*21410103050"# + ); + let sheet2 = + format!(r#"1"#); + let root_rels = rels(&[("rId1", "officeDocument", "xl/workbook.xml"), ("rId2", "custom-properties", "docProps/custom.xml")]); + let wb_rels = rels(&[ + ("rId1", "worksheet", "worksheets/sheet1.xml"), + ("rId2", "worksheet", "worksheets/sheet2.xml"), + ("rId3", "externalLink", "externalLinks/externalLink1.xml"), + ("rId4", "customXml", "../customXml/item1.xml"), + ("rId5", "connections", "connections.xml"), + ]); + let sheet1_rels = rels(&[ + ("rId1", "printerSettings", "../printerSettings/printerSettings1.bin"), + ("rId2", "vmlDrawing", "../drawings/vmlDrawing1.vml"), + ("rId3", "oleObject", "../embeddings/oleObject1.bin"), + ("rId4", "image", "../media/image1.emf"), + ("rId5", "http://schemas.openxmlformats.org/officeDocument/2006/relationships/ctrlProp", "../ctrlProps/ctrlProp1.xml"), + ("rId6", "queryTable", "../queryTables/queryTable1.xml"), + ("rId7", "comments", "../comments1.xml"), + ]); + let custom_rels = rels(&[("rId1", "customXmlProps", "itemProps1.xml")]); + let link_rels = rels(&[("rId1", "externalLinkPath", "file:///C:/never/opened/Book2.xlsx")]); + let vml_rels = rels(&[("rId1", "image", "../media/image1.emf")]); + let item_props = r#""#; + make_zip(&[ + ("[Content_Types].xml", CONTENT_TYPES.as_bytes()), + ("_rels/.rels", root_rels.as_bytes()), + ("docProps/custom.xml", CUSTOM_PROPS.as_bytes()), + ("xl/workbook.xml", workbook.as_bytes()), + ("xl/_rels/workbook.xml.rels", wb_rels.as_bytes()), + ("xl/worksheets/sheet1.xml", sheet1.as_bytes()), + ("xl/worksheets/sheet2.xml", sheet2.as_bytes()), + ("xl/worksheets/_rels/sheet1.xml.rels", sheet1_rels.as_bytes()), + ("xl/printerSettings/printerSettings1.bin", PRINTER), + ("xl/drawings/vmlDrawing1.vml", VML.as_bytes()), + ("xl/drawings/_rels/vmlDrawing1.vml.rels", vml_rels.as_bytes()), + ("xl/embeddings/oleObject1.bin", OLE), + ("xl/media/image1.emf", EMF), + ("xl/ctrlProps/ctrlProp1.xml", CTRL_PROP.as_bytes()), + ("xl/queryTables/queryTable1.xml", QUERY_TABLE.as_bytes()), + ("xl/comments1.xml", COMMENTS.as_bytes()), + ("customXml/item1.xml", CUSTOM_XML.as_bytes()), + ("customXml/_rels/item1.xml.rels", custom_rels.as_bytes()), + ("customXml/itemProps1.xml", item_props.as_bytes()), + ("xl/externalLinks/externalLink1.xml", EXTERNAL_LINK.as_bytes()), + ("xl/externalLinks/_rels/externalLink1.xml.rels", link_rels.as_bytes()), + ("xl/connections.xml", CONNECTIONS.as_bytes()), + ]) +} + +/// An unpacked package: parts by name, content types, resolved relationships. +struct Unpacked { + parts: HashMap>, +} + +/// (id, type, resolved target, external) +type RelList = Vec<(String, String, String, bool)>; + +impl Unpacked { + fn new(bytes: &[u8]) -> Unpacked { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut parts = HashMap::new(); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let mut v = vec![]; + f.read_to_end(&mut v).unwrap(); + parts.insert(f.name().to_string(), v); + } + Unpacked { parts } + } + fn text(&self, name: &str) -> String { + String::from_utf8(self.parts.get(name).unwrap_or_else(|| panic!("no part {name}")).clone()).unwrap() + } + fn rels(&self, part: &str) -> RelList { + let (dir, file) = split_dir(part); + let name = if dir.is_empty() { format!("_rels/{file}.rels") } else { format!("{dir}/_rels/{file}.rels") }; + let Some(b) = self.parts.get(&name) else { return vec![] }; + let root = xml::parse(b).unwrap(); + root.kids("Relationship") + .map(|r| { + let ext = r.attr("TargetMode") == Some("External"); + let t = r.attr("Target").unwrap(); + (r.attr("Id").unwrap().into(), r.attr("Type").unwrap().into(), if ext { t.into() } else { resolve(dir, t) }, ext) + }) + .collect() + } + fn target(&self, part: &str, id: &str) -> (String, String) { + let r = self.rels(part).into_iter().find(|r| r.0 == id).unwrap_or_else(|| panic!("{part}: no relationship {id}")); + (r.1, r.2) + } + /// Every part has a content type, every internal relationship target exists, rel ids are + /// unique per part. + fn assert_valid(&self) { + let types = xml::parse(self.parts.get("[Content_Types].xml").unwrap()).unwrap(); + let defaults: Vec = types.kids("Default").map(|d| d.attr("Extension").unwrap().to_ascii_lowercase()).collect(); + let overrides: Vec = types.kids("Override").map(|d| normalize(d.attr("PartName").unwrap())).collect(); + let mut seen_override = std::collections::HashSet::new(); + for o in &overrides { + assert!(seen_override.insert(o.clone()), "duplicate override {o}"); + assert!(self.parts.keys().any(|p| normalize(p) == *o), "override for a missing part {o}"); + } + for name in self.parts.keys().filter(|n| *n != "[Content_Types].xml") { + let ext = name.rsplit('.').next().unwrap().to_ascii_lowercase(); + assert!(overrides.contains(&normalize(name)) || defaults.contains(&ext), "{name} has no content type"); + if name.ends_with(".rels") { + let source = name.replace("_rels/", "").trim_end_matches(".rels").to_string(); + let rels = self.rels(&source); + let mut ids = std::collections::HashSet::new(); + for (id, _, target, ext) in rels { + assert!(ids.insert(id.clone()), "{name}: duplicate id {id}"); + if !ext { + assert!(self.parts.keys().any(|p| normalize(p) == normalize(&target)), "{name}: {id} → missing {target}"); + } + } + } + } + } + /// The part holding exactly `data`. + fn find(&self, data: &[u8]) -> Vec<&String> { + self.parts.iter().filter(|(_, v)| v.as_slice() == data).map(|(k, _)| k).collect() + } +} + +/// The value of attribute `attr` of the first `` in `x` (`r:id` of ` String { + let i = x.find(&format!("<{tag} ")).unwrap_or_else(|| panic!("no <{tag}> in {x}")); + let head = &x[i..i + x[i..].find('>').unwrap()]; + let j = head.find(&format!(" {attr}=\"")).unwrap_or_else(|| panic!("no {attr} in {head}")) + attr.len() + 3; + head[j..j + head[j..].find('"').unwrap()].to_string() +} + +fn sheet_part(u: &Unpacked, sheet: &str) -> String { + let wb = u.text("xl/workbook.xml"); + let i = wb.find(&format!(", its path kept as an external target. + let wbx = u.text("xl/workbook.xml"); + let link_id = attr_of(&wbx, "externalReference", "r:id"); + let (ty, link) = u.target("xl/workbook.xml", &link_id); + assert!(ty.ends_with("/externalLink")); + assert_eq!(u.parts.get(&link).unwrap(), EXTERNAL_LINK.as_bytes()); + let path = u.rels(&link); + assert_eq!(path.len(), 1); + assert_eq!((path[0].0.as_str(), path[0].2.as_str(), path[0].3), ("rId1", "file:///C:/never/opened/Book2.xlsx", true)); + assert!(wbx.find("").unwrap() < wbx.find("[1]Sheet1!$A$1"), "{sx}"); + assert!(sx.contains("SUM([1]Sheet1!A1:A3)*2"), "{sx}"); + // Printer settings (#324). + let ps = attr_of(&sx, "pageSetup", "r:id"); + let (ty, target) = u.target(&sheet, &ps); + assert!(ty.ends_with("/printerSettings")); + assert_eq!(u.parts.get(&target).unwrap(), PRINTER); + // The OLE object and its preview image, re-linked. + let ole = attr_of(&sx, "oleObject", "r:id"); + assert_eq!(u.parts.get(&u.target(&sheet, &ole).1).unwrap(), OLE); + let preview = attr_of(&sx, "objectPr", "r:id"); + assert_eq!(u.parts.get(&u.target(&sheet, &preview).1).unwrap(), EMF); + // Namespace declarations travel with the fragment. + assert_eq!(attr_of(&sx, "mc:AlternateContent", "xmlns:mc"), "http://schemas.openxmlformats.org/markup-compatibility/2006"); + assert_eq!(attr_of(&sx, "mc:AlternateContent", "xmlns:x14"), "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main"); + // The form control. + let ctrl = attr_of(&sx, "control", "r:id"); + assert_eq!(u.parts.get(&u.target(&sheet, &ctrl).1).unwrap(), CTRL_PROP.as_bytes()); + // Schema order: pageSetup < legacyDrawing < oleObjects < controls. + let pos = |t: &str| sx.find(t).unwrap_or_else(|| panic!("no {t}")); + assert!(pos(" = a.parts.iter().map(|p| &*p.data).collect(); + let mut db: Vec<&[u8]> = b.parts.iter().map(|p| &*p.data).collect(); + da.sort(); + db.sort(); + assert_eq!(da, db); + assert_eq!(a.elements.len(), b.elements.len()); + assert_eq!(wb.preserved.as_deref().unwrap().parts.len(), wb2.preserved.as_deref().unwrap().parts.len()); + u.assert_valid(); +} + +#[test] +fn edited_workbooks_keep_consistent_relationships() { + let (mut wb, _) = read_xlsx(&rich_package()).unwrap(); + // Insert a sheet before, rename the sheet with controls, drop its note, edit a cell. + wb.sheets.insert(0, Arc::new(Sheet::new("New"))); + let s = wb.sheet_mut(1).unwrap(); + s.name = "Renamed".into(); + s.comments.clear(); + s.set_value(gridcraft_core::CellRef::new(5, 5), gridcraft_core::Value::Number(3.0)); + let u = Unpacked::new(&write_xlsx(&wb).unwrap()); + u.assert_valid(); + let sheet = sheet_part(&u, "Renamed"); + assert_eq!(sheet, "xl/worksheets/sheet2.xml"); + let sx = u.text(&sheet); + assert_eq!(u.parts.get(&u.target(&sheet, &attr_of(&sx, "pageSetup", "r:id")).1).unwrap(), PRINTER); + assert_eq!(u.parts.get(&u.target(&sheet, &attr_of(&sx, "control", "r:id")).1).unwrap(), CTRL_PROP.as_bytes()); + // Without notes the kept VML is written on its own, still without the old note. + let vml = u.text(&u.target(&sheet, &attr_of(&sx, "legacyDrawing", "r:id")).1); + assert!(vml.contains("_x0000_s1026") && !vml.contains("ObjectType=\"Note\"")); + assert!(!u.text("xl/worksheets/sheet1.xml").contains("pageSetup orientation=\"landscape\" r:id")); + + // Deleting the sheet drops its content; workbook-level content stays. + wb.sheets.remove(1); + wb.names.clear(); + let u = Unpacked::new(&write_xlsx(&wb).unwrap()); + u.assert_valid(); + for d in [PRINTER, OLE, EMF, QUERY_TABLE.as_bytes(), CTRL_PROP.as_bytes()] { + assert!(u.find(d).is_empty()); + } + for d in [CUSTOM_XML.as_bytes(), EXTERNAL_LINK.as_bytes(), CONNECTIONS.as_bytes(), CUSTOM_PROPS.as_bytes()] { + assert_eq!(u.find(d).len(), 1); + } +} + +/// #324: an openpyxl-style workbook whose sheet carries printer settings. +#[test] +fn printer_settings_and_page_setup_id_survive_save() { + let ct = r#""#; + let bin: Vec = (0..=255u8).cycle().take(1024).collect(); + let pkg = super::minimal( + r#"42.1"#, + &[("xl/worksheets/_rels/sheet1.xml.rels", &rels(&[("rIdPs1", "printerSettings", "../printerSettings/printerSettings1.bin")]))], + "", + "", + ); + // Swap in content types that know `.bin`, and add the binary part. + let mut u = Unpacked::new(&pkg); + u.parts.insert("[Content_Types].xml".into(), ct.as_bytes().to_vec()); + u.parts.insert("xl/printerSettings/printerSettings1.bin".into(), bin.clone()); + let list: Vec<(&str, &[u8])> = u.parts.iter().map(|(k, v)| (k.as_str(), v.as_slice())).collect(); + let (wb, rep) = read_xlsx(&make_zip(&list)).unwrap(); + assert!(rep.warnings.iter().all(|w| !w.contains("printer")), "{:?}", rep.warnings); + let out = Unpacked::new(&write_xlsx(&wb).unwrap()); + out.assert_valid(); + let sx = out.text("xl/worksheets/sheet1.xml"); + assert!(sx.contains("orientation=\"landscape\"")); + let (ty, target) = out.target("xl/worksheets/sheet1.xml", &attr_of(&sx, "pageSetup", "r:id")); + assert!(ty.ends_with("/printerSettings")); + assert_eq!(target, "xl/printerSettings/printerSettings1.bin"); + assert_eq!(out.parts.get(&target).unwrap(), &bin); +} + +#[test] +fn damaged_kept_content_is_dropped_without_failing_the_read() { + let ct = CONTENT_TYPES.replace("", r#""#); + let workbook = format!( + r#""# + ); + let sheet = format!( + r#""# + ); + let wb_rels = rels(&[ + ("rId1", "worksheet", "worksheets/sheet1.xml"), + ("rId3", "externalLink", "externalLinks/externalLink1.xml"), + // A missing part. + ("rId4", "customXml", "../customXml/missing.xml"), + // A cycle: item1 → itemProps1 → item1. + ("rId5", "customXml", "../customXml/item1.xml"), + // Kept content that points into a part GridCraft rewrites. + ("rId6", "customXml", "../customXml/item2.xml"), + // Damaged rels behind a kept part. + ("rId7", "connections", "connections.xml"), + ]); + let cyclic = rels(&[("rId1", "customXmlProps", "itemProps1.xml")]); + let back = rels(&[("rId1", "customXml", "item1.xml")]); + let into_sheet = rels(&[("rId1", "worksheet", "../xl/worksheets/sheet1.xml")]); + let sheet_rels = rels(&[("rId1", "printerSettings", "../printerSettings/missing.bin")]); + let bytes = make_zip(&[ + ("[Content_Types].xml", ct.as_bytes()), + ("_rels/.rels", super::ROOT_RELS.as_bytes()), + ("xl/workbook.xml", workbook.as_bytes()), + ("xl/_rels/workbook.xml.rels", wb_rels.as_bytes()), + ("xl/worksheets/sheet1.xml", sheet.as_bytes()), + ("xl/worksheets/_rels/sheet1.xml.rels", sheet_rels.as_bytes()), + ("xl/externalLinks/externalLink1.xml", EXTERNAL_LINK.as_bytes()), + ("customXml/item1.xml", CUSTOM_XML.as_bytes()), + ("customXml/_rels/item1.xml.rels", cyclic.as_bytes()), + ("customXml/itemProps1.xml", b"

"), + ("customXml/_rels/itemProps1.xml.rels", back.as_bytes()), + ("customXml/item2.xml", b""), + ("customXml/_rels/item2.xml.rels", into_sheet.as_bytes()), + ("xl/connections.xml", CONNECTIONS.as_bytes()), + ("xl/_rels/connections.xml.rels", b" = p.parts.iter().map(|q| q.name.as_str()).collect(); + names.sort(); + assert_eq!(names, ["customXml/item1.xml", "customXml/itemProps1.xml"]); + assert!(p.elements.is_empty()); + assert!(wb.sheet(0).unwrap().preserved.is_none()); + let u = Unpacked::new(&write_xlsx(&wb).unwrap()); + u.assert_valid(); + assert!(!u.text("xl/workbook.xml").contains("externalReferences")); +} + +#[test] +fn oversized_kept_parts_are_dropped() { + let big = vec![0u8; crate::passthrough::MAX_KEPT_PART as usize + 1]; + let ct = CONTENT_TYPES; + let wb_rels = rels(&[("rId1", "worksheet", "worksheets/sheet1.xml"), ("rId2", "customXml", "../customXml/item1.xml")]); + let workbook = format!( + r#""# + ); + let sheet = format!(r#""#); + let bytes = make_zip(&[ + ("[Content_Types].xml", ct.as_bytes()), + ("_rels/.rels", super::ROOT_RELS.as_bytes()), + ("xl/workbook.xml", workbook.as_bytes()), + ("xl/_rels/workbook.xml.rels", wb_rels.as_bytes()), + ("xl/worksheets/sheet1.xml", sheet.as_bytes()), + ("customXml/item1.xml", &big), + ]); + drop(big); + let (wb, rep) = read_xlsx(&bytes).unwrap(); + assert!(wb.preserved.is_none()); + assert!(rep.warnings.iter().any(|w| w.contains("customXml") && w.contains("dropped")), "{:?}", rep.warnings); +} diff --git a/crates/xlsx/src/write.rs b/crates/xlsx/src/write.rs index 8af3df1e8..96a803a39 100644 --- a/crates/xlsx/src/write.rs +++ b/crates/xlsx/src/write.rs @@ -49,6 +49,12 @@ impl Rels { self.items.push((id.clone(), ty.to_string(), target.to_string(), false)); id } + /// Adds a relationship with its complete type URI, internal or external (kept content). + pub fn add_raw(&mut self, ty: &str, target: &str, external: bool) -> String { + let id = format!("rId{}", self.items.len() + 1); + self.items.push((id.clone(), crate::xml::esc_attr(ty), target.to_string(), external)); + id + } pub fn is_empty(&self) -> bool { self.items.is_empty() } @@ -106,6 +112,8 @@ pub struct Out { pub dynamic_arrays: bool, pub needs_calc: bool, pub cell_pictures: crate::richdata::PicturesOut, + /// Kept (pass-through) parts written so far. + pub kept: crate::passthrough::KeptOut, } impl Out { @@ -158,6 +166,7 @@ pub fn write_xlsx(wb: &Workbook) -> Result, IoError> { dynamic_arrays: false, needs_calc: false, cell_pictures: Default::default(), + kept: Default::default(), }; out.default_type("rels", "application/vnd.openxmlformats-package.relationships+xml"); out.default_type("xml", "application/xml"); @@ -247,6 +256,7 @@ pub fn write_xlsx(wb: &Workbook) -> Result, IoError> { w.push_str(""); } let _ = write!(w, "{sheets_xml}"); + w.push_str(&crate::passthrough::write_workbook(wb, &mut wb_rels, &mut out)); if !names.is_empty() { let _ = write!(w, "{names}"); } @@ -329,6 +339,7 @@ pub fn write_xlsx(wb: &Workbook) -> Result, IoError> { root.add("officeDocument", "xl/workbook.xml"); root.add("core-properties", "docProps/core.xml"); root.add("extended-properties", "docProps/app.xml"); + crate::passthrough::write_package_rels(&mut root, &mut out); out.part("_rels/.rels", None, root.xml().into_bytes()); // Content types first, then the parts in creation order. diff --git a/crates/xlsx/src/xml.rs b/crates/xlsx/src/xml.rs index efa5cdcb1..3a4760e27 100644 --- a/crates/xlsx/src/xml.rs +++ b/crates/xlsx/src/xml.rs @@ -82,7 +82,7 @@ pub fn rich_text(e: &El) -> String { } /// Converts input bytes to UTF-8 XML text (handles BOMs and UTF-16). -fn to_utf8(xml: &[u8]) -> std::borrow::Cow<'_, [u8]> { +pub(crate) fn to_utf8(xml: &[u8]) -> std::borrow::Cow<'_, [u8]> { if let Some(rest) = xml.strip_prefix(&[0xEF, 0xBB, 0xBF]) { return std::borrow::Cow::Borrowed(rest); } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index eeb2dd9b4..8959cc023 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (created; every format Excel reads or writes, against the code on main) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (XLSX pass-through of unmodelled parts, #330) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Every format Excel opens or saves, with GridCraft's read and write support, fidelity and tests. Part of [target-app-parity.md](target-app-parity.md). **File formats: ~50% ready, 80–120 h** @@ -15,7 +15,7 @@ support comes from `crates/engine/src/io.rs` (`FileKind`), `crates/xlsx/src` and | Format | Excel | GridCraft read | GridCraft write | Fidelity / notes | Tests | |---|---|---|---|---|---| -| `.xlsx` Excel Workbook (transitional) | R/W, native | yes | yes | Cells, shared/array/dynamic formulas, styles, themes, merges, CF, validation, tables, comments and notes, hyperlinks, names, freeze panes, outline, print settings, pictures, shapes, charts (simplified, see [chart-parity.md](chart-parity.md)), sparklines, PivotTables (caches and tables), protection, calc settings. **Not preserved:** unmodelled parts and extensions (form controls, slicer caches, timelines, external links, OLE objects, custom XML, data model, query tables, chart details). Never checked in Excel across a corpus: repair prompts unknown. 100 MB files fail (#175) | `roundtrip.rs`, `fixtures.rs`, `malformed.rs`, `pivot_tests.rs` | +| `.xlsx` Excel Workbook (transitional) | R/W, native | yes | yes | Cells, shared/array/dynamic formulas, styles, themes, merges, CF, validation, tables, comments and notes, hyperlinks, names, freeze panes, outline, print settings, pictures, shapes, charts (simplified, see [chart-parity.md](chart-parity.md)), sparklines, PivotTables (caches and tables), protection, calc settings. **Kept unchanged on save (pass-through, `passthrough.rs`):** custom document properties, ribbon customisation and add-in task panes, custom XML, links to external workbooks (never followed or refreshed), data connections, sheet query tables, printer settings (#324), form controls, ActiveX controls, OLE objects and sheet backgrounds (with their VML shapes); dropped when their sheet is deleted or their parts point at content GridCraft rewrites. **Not preserved:** slicers and slicer caches, timelines, data model, query tables attached to tables, sheet and workbook `extLst` entries we don't model, chart details; Strict Open XML files keep nothing. Never checked in Excel across a corpus: repair prompts unknown. 100 MB files fail (#175) | `roundtrip.rs`, `fixtures.rs`, `malformed.rs`, `pivot_tests.rs` | | `.xlsx` Strict Open XML | R/W | yes | no (writes transitional) | Excel also saves transitional by default | fixtures | | `.xlsm` Macro-Enabled Workbook | R/W | yes | yes, **without macros** | VBA project dropped with a warning; saving as `.xlsm` loses the macros (#6) | — | | `.xltx` / `.xltm` templates | R/W | yes | partial | Saved with the workbook content type, not the template one; "new from template" behaviour missing | — | @@ -62,7 +62,7 @@ refresh. Counted under features (Get & Transform, ~10% ready), not here. | Work | Hours | |---|---| | Real-world XLSX corpus run (craftrules test-corpora standard), and confirm Excel opens our output with no repair prompt (owner runs Excel) | 15–25 | -| Preserve unmodelled XLSX parts and extensions on round trip (pass-through with relationship fix-up) | 10–15 | +| Preserve the rest of the unmodelled XLSX content (slicers, timelines, data model, table query tables, `extLst` entries); pass-through landed for the rest (#330) | 6–10 | | Streaming XLSX reader for very large files (#175), part-size limits that degrade instead of failing | 10–15 | | Legacy `.xls` (BIFF8) import: cells, formulas, styles, names; then write | 20–30 | | Encrypted workbooks: open with password prompt, save with password (#11) | 8–12 | @@ -74,5 +74,6 @@ refresh. Counted under features (Get & Transform, ~10% ready), not here. | Date | Change | Summary | |---|---|---| +| 2026-10-11 | minor | XLSX pass-through: custom XML, external links, connections, query tables, printer settings, form/ActiveX controls, OLE objects, sheet backgrounds kept on save (#330, #324) | | 2026-10-10 | minor | Encrypted and `.xls` files are now recognised and refused with a message (#17) | | 2026-10-10 | major | Created: every Excel format with our read/write support; XLSB and ODS values-only imports recorded; `.xls`, encryption and large files flagged as beta gates | diff --git a/docs/gaps.md b/docs/gaps.md index aa752b8d8..de3faba9d 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Gaps -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (created from the full re-measure against Excel for Mac 16.113.4 and the issue tracker) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (§3 narrowed by XLSX pass-through, #330) · **Target:** Microsoft Excel (Microsoft 365) Every known shortfall against Excel, one entry each, ranked by user impact (what blocks someone using GridCraft for real work first). This is the work list: pick from the top, check the open @@ -33,12 +33,14 @@ No alpha blockers: every core workflow passes the [alpha gate](roadmap.md#alpha- - **Impact:** finance and data users routinely have big workbooks. - **Estimate:** 25–40 h. [target-app-parity.md](target-app-parity.md#performance), [hardware-parity.md](hardware-parity.md) -### 3. Unmodelled XLSX content is dropped on save (FF) -- **Missing:** pass-through of parts and extensions we don't model (form controls, slicers, - timelines, external links, OLE objects, custom XML, query tables, data model, chart details). -- **Evidence:** reader warns and skips (`crates/xlsx/src/read.rs`); external links "not supported". -- **Impact:** open-edit-save silently simplifies colleagues' workbooks. -- **Estimate:** 10–15 h. [file-format-parity.md](file-format-parity.md) +### 3. Some unmodelled XLSX content is dropped on save (FF) +- **Missing:** slicers and their caches, timelines, the data model, query tables attached to + tables, unmodelled `extLst` entries, chart details. Pass-through (#330) keeps custom XML, + external links, connections, sheet query tables, printer settings, form and ActiveX controls, + OLE objects and sheet backgrounds; never checked in Excel (repair prompts unknown). +- **Evidence:** `crates/xlsx/src/passthrough.rs` (what is kept, and why the rest isn't). +- **Impact:** open-edit-save still simplifies workbooks with slicers, timelines or Power Pivot. +- **Estimate:** 6–10 h. [file-format-parity.md](file-format-parity.md) ### 4. Charts lose type and options on round trip; shallow chart model (F, FF) - **Missing:** kinds that survive XLSX (stock, combo, histogram, box, waterfall, funnel, treemap, @@ -149,6 +151,7 @@ No alpha blockers: every core workflow passes the [alpha gate](roadmap.md#alpha- | Date | Change | Summary | |---|---|---| +| 2026-10-11 | minor | §3: XLSX pass-through keeps custom XML, external links, connections, query tables, printer settings, controls and OLE objects (#330, #324) | | 2026-10-10 | minor | Updated for the 10-10 merges (#17, #31, #38, #62, #82, #104, #116) and the new chart resize bug (#190) | | 2026-10-10 | minor | Noted that no gap blocks the alpha gate | | 2026-10-10 | major | Created: 24 ranked gaps with evidence, from the full re-measure and the 60 open issues | From e52fc4aa08b5f1b6456507566e3eaf2ede6a9b10 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 Oct 2026 13:05:45 -0400 Subject: [PATCH 2/2] Fix-up #333: escape kept content types and charge kept VML to the kept total Content types (and extensions) of kept parts come from the input file and were written into [Content_Types].xml unescaped, so a quote or angle bracket injected XML; they are now attribute-escaped. Each sheet re-read and kept its VML drawing (up to 16 MB) outside MAX_KEPT_TOTAL, so many sheets sharing one highly compressed drawing amplified memory; the drawing is now read within and charged to the remaining kept-bytes allowance. Co-Authored-By: Claude Opus 5.5 --- crates/xlsx/src/passthrough.rs | 30 +++++++++- crates/xlsx/src/tests/passthrough_tests.rs | 65 ++++++++++++++++++++-- crates/xlsx/src/write.rs | 5 +- 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/crates/xlsx/src/passthrough.rs b/crates/xlsx/src/passthrough.rs index 187009bca..f121977e2 100644 --- a/crates/xlsx/src/passthrough.rs +++ b/crates/xlsx/src/passthrough.rs @@ -32,6 +32,7 @@ use quick_xml::events::{BytesStart, Event}; use quick_xml::name::QName; use quick_xml::{Reader, Writer}; +use crate::IoError; use crate::package::{self, normalize}; use crate::read::Ctx; use crate::write::{NS_REL, Out, Rels, XML_DECL}; @@ -54,6 +55,23 @@ const MAX_FRAGMENT: usize = 4 * 1024 * 1024; /// Deepest element nesting accepted in a kept fragment. const MAX_FRAGMENT_DEPTH: usize = 256; +#[cfg(test)] +thread_local! { + /// A smaller kept-bytes allowance for tests on this thread. + pub(crate) static KEPT_TOTAL_FOR_TEST: std::cell::Cell = const { std::cell::Cell::new(MAX_KEPT_TOTAL) }; +} + +fn kept_total() -> usize { + #[cfg(test)] + { + KEPT_TOTAL_FOR_TEST.with(std::cell::Cell::get) + } + #[cfg(not(test))] + { + MAX_KEPT_TOTAL + } +} + const NS_REL_STRICT: &str = "http://purl.oclc.org/ooxml/officeDocument/relationships"; const CT_VML: &str = "application/vnd.openxmlformats-officedocument.vmlDrawing"; @@ -194,7 +212,7 @@ impl KeptReader { strict: false, defaults: HashMap::new(), overrides: HashMap::new(), - budget: MAX_KEPT_TOTAL, + budget: kept_total(), parts_left: MAX_KEPT_PARTS, }; if let Ok(root) = raw_rels(cx, "") @@ -446,11 +464,19 @@ impl KeptReader { seen: &mut HashSet, ) -> Result<(PreservedPart, Vec), String> { let rel = rid.and_then(|id| rels.iter().find(|r| r.id == id && !r.external)).ok_or("the sheet has no VML drawing")?; - let data = match cx.pkg.read_limited(&rel.target, MAX_META) { + // Charged to the kept total like any other part: many sheets can point at one highly + // compressed VML drawing, and each keeps its own copy. + if self.budget == 0 { + return Err("it is too large".into()); + } + let limit = MAX_META.min(u64::try_from(self.budget).unwrap_or(u64::MAX)); + let data = match cx.pkg.read_limited(&rel.target, limit) { Ok(Some(d)) => d, Ok(None) => return Err(format!("{} is missing", rel.target)), + Err(IoError::TooLarge(_)) => return Err("it is too large".into()), Err(e) => return Err(e.to_string()), }; + self.budget = self.budget.saturating_sub(data.len()); let text = String::from_utf8(data).map_err(|_| "the VML drawing is not UTF-8".to_string())?; let text = strip_notes(&text); let vrels = raw_rels(cx, &rel.target)?; diff --git a/crates/xlsx/src/tests/passthrough_tests.rs b/crates/xlsx/src/tests/passthrough_tests.rs index 10b2681d7..4cde9d69e 100644 --- a/crates/xlsx/src/tests/passthrough_tests.rs +++ b/crates/xlsx/src/tests/passthrough_tests.rs @@ -45,14 +45,23 @@ const VML: &str = r##" Vec { let workbook = format!( r#""# ); let sheet1 = format!( r#"[1]Sheet1!$A$17SUM([1]Sheet1!A1:A3)*21410103050"# ); - let sheet2 = - format!(r#"1"#); + let sheet2 = if twin { + sheet1.clone() + } else { + format!(r#"1"#) + }; let root_rels = rels(&[("rId1", "officeDocument", "xl/workbook.xml"), ("rId2", "custom-properties", "docProps/custom.xml")]); let wb_rels = rels(&[ ("rId1", "worksheet", "worksheets/sheet1.xml"), @@ -74,8 +83,8 @@ fn rich_package() -> Vec { let link_rels = rels(&[("rId1", "externalLinkPath", "file:///C:/never/opened/Book2.xlsx")]); let vml_rels = rels(&[("rId1", "image", "../media/image1.emf")]); let item_props = r#""#; - make_zip(&[ - ("[Content_Types].xml", CONTENT_TYPES.as_bytes()), + let mut files: Vec<(&str, &[u8])> = vec![ + ("[Content_Types].xml", content_types.as_bytes()), ("_rels/.rels", root_rels.as_bytes()), ("docProps/custom.xml", CUSTOM_PROPS.as_bytes()), ("xl/workbook.xml", workbook.as_bytes()), @@ -97,7 +106,11 @@ fn rich_package() -> Vec { ("xl/externalLinks/externalLink1.xml", EXTERNAL_LINK.as_bytes()), ("xl/externalLinks/_rels/externalLink1.xml.rels", link_rels.as_bytes()), ("xl/connections.xml", CONNECTIONS.as_bytes()), - ]) + ]; + if twin { + files.push(("xl/worksheets/_rels/sheet2.xml.rels", sheet1_rels.as_bytes())); + } + make_zip(&files) } /// An unpacked package: parts by name, content types, resolved relationships. @@ -428,3 +441,45 @@ fn oversized_kept_parts_are_dropped() { assert!(wb.preserved.is_none()); assert!(rep.warnings.iter().any(|w| w.contains("customXml") && w.contains("dropped")), "{:?}", rep.warnings); } + +#[test] +fn kept_content_types_are_escaped_on_save() { + // A content type holding `"` and `<` (legal once unescaped by the XML reader) must not break + // out of the attribute in the saved [Content_Types].xml. + let hostile = r#"application/x-test" PartName="/x"/><Default Extension="evil"#; + let ct = CONTENT_TYPES.replace("application/vnd.openxmlformats-officedocument.customXmlProperties+xml", hostile); + let (wb, rep) = read_xlsx(&rich_package_with(&ct, false)).unwrap(); + assert!(wb.preserved.is_some(), "{:?}", rep.warnings); + let u = Unpacked::new(&write_xlsx(&wb).unwrap()); + let types = xml::parse(u.text("[Content_Types].xml").as_bytes()).unwrap(); + assert!(!types.kids("Default").any(|d| d.attr("Extension") == Some("evil"))); + let item = types.kids("Override").find(|o| o.attr("PartName") == Some("/customXml/itemProps1.xml")).unwrap(); + assert_eq!(item.attr("ContentType"), Some(r#"application/x-test" PartName="/x"/>() + }; + // Two sheets share one VML drawing: each would keep its own copy. With room for one sheet's + // content only, the second sheet's VML (and the controls drawn from it) is dropped. + crate::passthrough::KEPT_TOTAL_FOR_TEST.with(|c| c.set(one + VML.len() / 2)); + let read = read_xlsx(&rich_package_with(CONTENT_TYPES, true)); + crate::passthrough::KEPT_TOTAL_FOR_TEST.with(|c| c.set(crate::passthrough::MAX_KEPT_TOTAL)); + let (wb, rep) = read.unwrap(); + let kept = wb.preserved.as_ref().map_or(0, |p| p.byte_len()) + + wb.sheets.iter().filter_map(|s| s.preserved.as_ref()).map(|p| p.byte_len()).sum::(); + assert!(kept <= one + VML.len() / 2, "kept {kept} bytes"); + assert!(wb.sheet(0).unwrap().preserved.as_ref().is_some_and(|p| p.vml.is_some())); + assert!(!wb.sheet(1).unwrap().preserved.as_ref().is_some_and(|p| p.vml.is_some())); + assert!(rep.warnings.iter().any(|w| w.contains("\"Other\"") && w.contains("too large")), "{:?}", rep.warnings); + // Both sheets keep their drawings when there is room. + let (wb, _) = read_xlsx(&rich_package_with(CONTENT_TYPES, true)).unwrap(); + assert!(wb.sheets.iter().all(|s| s.preserved.as_ref().is_some_and(|p| p.vml.is_some()))); +} diff --git a/crates/xlsx/src/write.rs b/crates/xlsx/src/write.rs index 95fbed9c6..dc2ddc484 100644 --- a/crates/xlsx/src/write.rs +++ b/crates/xlsx/src/write.rs @@ -375,10 +375,11 @@ pub fn write_xlsx(wb: &Workbook) -> Result, IoError> { // Content types first, then the parts in creation order. let mut ct = format!("{XML_DECL}"); for (ext, t) in &out.defaults { - let _ = write!(ct, ""); + // Kept parts bring their content types (and extensions) from the input file: escape them. + let _ = write!(ct, "", esc_attr(ext), esc_attr(t)); } for (part, t) in &out.overrides { - let _ = write!(ct, "", esc_attr(part)); + let _ = write!(ct, "", esc_attr(part), esc_attr(t)); } ct.push_str(""); let mut zip = ZipOut::new();