From 4f7dd932681d5745082fc573ea807486c97409c3 Mon Sep 17 00:00:00 2001 From: aiongg Date: Mon, 26 Oct 2026 21:00:03 +0000 Subject: [PATCH] refactor(engine): move cmd/variablecmds.rs items to cmd/variablecmds/{csv,library}.rs --- crates/engine/src/cmd/variablecmds.rs | 645 +----------------- crates/engine/src/cmd/variablecmds/csv.rs | 351 ++++++++++ crates/engine/src/cmd/variablecmds/library.rs | 298 ++++++++ 3 files changed, 655 insertions(+), 639 deletions(-) create mode 100644 crates/engine/src/cmd/variablecmds/csv.rs create mode 100644 crates/engine/src/cmd/variablecmds/library.rs diff --git a/crates/engine/src/cmd/variablecmds.rs b/crates/engine/src/cmd/variablecmds.rs index 0106e9895..be5fec64a 100644 --- a/crates/engine/src/cmd/variablecmds.rs +++ b/crates/engine/src/cmd/variablecmds.rs @@ -21,24 +21,14 @@ use super::fileio::{absolute_path, read_file_head, write_file}; use super::textedit::set_plain_text; use super::*; +mod csv; +mod library; + +use csv::{MAX_GRAPH_CSV, dataset_export, dataset_import}; +use library::load_library; + /// The longest variable or dataset name, in characters (names arrive from MCP and files). const MAX_NAME: usize = 255; -/// Cap on a graph variable's CSV (MCP and files are untrusted). -const MAX_GRAPH_CSV: usize = 256 * 1024; - -/// Caps on a datasets CSV (paths and MCP payloads are untrusted). -const MAX_CSV_BYTES: usize = 1_048_576; -const MAX_CSV_ROWS: usize = 1_000; -const MAX_CSV_COLS: usize = 256; - -/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how -/// deep its elements nest, and the entities, variables and data sets it declares. -const MAX_LIBRARY_BYTES: usize = 8 << 20; -const MAX_LIBRARY_ENTITIES: usize = 64; -const MAX_LIBRARY_NODES: u32 = 2_000_000; -const MAX_LIBRARY_NESTING: usize = 256; -const MAX_LIBRARY_VARIABLES: usize = 256; -const MAX_LIBRARY_DATASETS: usize = 1_000; /// Lock Variables refuses New Variable, Delete Variable, Rename Variable (Variable Options), /// Make … Dynamic, an import whose header would define a variable and a Load Variable Library @@ -545,346 +535,6 @@ fn dataset_select(s: &mut Session, p: &Value) -> Result { apply_dataset(s, &var_name(p, "dataset.select")?) } -/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`. -fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result> { - let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) { - // One byte past the cap is enough to refuse the file below. - (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?, - (None, Some(text), ..) => text.as_bytes().to_vec(), - (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?, - (None, None, None, Some(csv)) => csv.as_bytes().to_vec(), - _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")), - }; - if bytes.len() > MAX_CSV_BYTES { - return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes"))); - } - Ok(bytes) -} - -/// A row of a datasets CSV: the file line it starts on, and its cells. -type CsvRow = (usize, Vec); - -/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that -/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside -/// quotes. -fn tab_separated(text: &str) -> bool { - // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds - // anything but white space yet. - let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false); - let mut chars = text.chars().peekable(); - while let Some(c) = chars.next() { - match c { - '"' if in_quotes => { - if chars.peek() == Some(&'"') { - chars.next(); - blank = false; - } else { - in_quotes = false; - } - } - '"' if start => in_quotes = true, - c if in_quotes => blank &= c.is_whitespace(), - ',' => return false, - '\t' => { - tab = true; - start = true; - continue; - } - '\r' => continue, - '\n' if blank => { - tab = false; - start = true; - continue; - } - '\n' => return tab, - c => blank &= c.is_whitespace(), - } - start = false; - } - tab -} - -/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated -/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside), -/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted -/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and -/// drops white space after the closing quote at the end of the cell. Blank lines are skipped. -fn parse_datasets_csv(text: &str, cmd: &str) -> Result> { - let text = text.strip_prefix('\u{feff}').unwrap_or(text); - let sep = if tab_separated(text) { '\t' } else { ',' }; - let (mut rows, mut row, mut cur) = (Vec::new(), Vec::::new(), String::new()); - // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the - // current row starts on. - let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize); - let mut chars = text.chars().peekable(); - loop { - let c = chars.next(); - match c { - Some('"') if in_quotes => { - if chars.peek() == Some(&'"') { - cur.push('"'); - chars.next(); - } else { - in_quotes = false; - closed = Some(cur.len()); - } - } - Some('"') if cur.is_empty() => in_quotes = true, - Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)), - Some('\r') if !in_quotes => {} - Some('\n') | None if !in_quotes => { - row.push(end_csv_cell(&mut cur, &mut closed)); - if row.iter().any(|c| !c.trim().is_empty()) { - rows.push((start, std::mem::take(&mut row))); - // +1 for the header. - if rows.len() > MAX_CSV_ROWS + 1 { - return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows"))); - } - } - row.clear(); - if c.is_none() { - return Ok(rows); - } - line += 1; - start = line; - } - None => return Err(bad(cmd, format!("line {line}: unclosed quote"))), - Some(c) => { - line += usize::from(c == '\n'); - cur.push(c); - } - } - if row.len() > MAX_CSV_COLS { - return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns"))); - } - } -} - -/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell -/// is trimmed; with one, the text before it is kept as written and the text after it is -/// trimmed at the end. -fn end_csv_cell(cur: &mut String, closed: &mut Option) -> String { - let cell = std::mem::take(cur); - match closed.take() { - Some(at) => match cell.split_at_checked(at) { - Some((quoted, after)) => format!("{quoted}{}", after.trim_end()), - None => cell, - }, - None => cell.trim().to_string(), - } -} - -fn csv_escape(s: &str) -> String { - if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() } -} - -/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive). -fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result { - match s.trim().to_ascii_lowercase().as_str() { - "true" | "1" | "yes" => Ok(true), - "false" | "0" | "no" => Ok(false), - _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))), - } -} - -fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result> { - // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and - // visibility cells are trimmed. - let t = if kind == VariableKind::Text { cell } else { cell.trim() }; - if t.is_empty() { - return Ok(None); - } - Ok(Some(match kind { - VariableKind::Text => DataValue::Text(t.to_string()), - VariableKind::Image => DataValue::Image(t.to_string()), - VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()), - VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))), - VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?), - })) -} - -fn value_to_cell(v: &DataValue) -> String { - match v { - DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(), - DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(), - } -} - -/// Build the CSV of every dataset (header `name` then variables in definition order). -fn datasets_to_csv(vars: &Variables) -> String { - let mut out = String::from("name"); - for v in &vars.variables { - out.push(','); - out.push_str(&csv_escape(&v.name)); - } - out.push('\n'); - for ds in &vars.datasets { - out.push_str(&csv_escape(&ds.name)); - for v in &vars.variables { - out.push(','); - if let Some(val) = ds.values.get(&v.name) { - out.push_str(&csv_escape(&value_to_cell(val))); - } - } - out.push('\n'); - } - out -} - -/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names, -/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables -/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a -/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text -/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new -/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused. -fn header_variable( - cell: &str, - column: usize, - kinds: &BTreeMap, - defined: &mut Vec, - cmd: &str, -) -> Result<(String, VariableKind)> { - let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind)); - if let Some(kind) = known(cell, defined) { - return Ok((cell.to_string(), kind)); - } - let (name, kind) = if let Some(rest) = cell.strip_prefix('#') { - (rest.trim(), VariableKind::Visibility) - } else if let Some(rest) = cell.strip_prefix('@') { - (rest.trim(), VariableKind::Image) - } else if cell.starts_with('%') { - return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read"))); - } else { - (cell, VariableKind::Text) - }; - if name.is_empty() { - return Err(bad(cmd, format!("column {column}: `{cell}` names no variable"))); - } - match known(name, defined) { - Some(k) if k == kind => Ok((name.to_string(), kind)), - Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))), - None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))), - None => { - defined.push(Variable { name: name.to_string(), kind }); - Ok((name.to_string(), kind)) - } - } -} - -/// Import datasets from CSV. A first header cell `name` makes the first column the dataset -/// names; the other headers name variables, as [`header_variable`] reads them, and the -/// variables the document doesn't define yet are defined, unbound. Without the `name` column, -/// every header names a variable and the rows take the first free names "Data Set 1", "Data -/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One -/// undo step. -fn dataset_import(s: &mut Session, p: &Value) -> Result { - const C: &str = "dataset.import"; - let bytes = datasets_csv_bytes(p, C)?; - let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?; - let rows = parse_datasets_csv(text, C)?; - let Some((_, header)) = rows.first() else { - return Err(bad(C, "CSV is empty")); - }; - let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name")); - let (kinds, existing_names) = { - let st = s.doc()?; - let kinds: BTreeMap = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect(); - let existing: BTreeSet = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect(); - (kinds, existing) - }; - let mut columns: Vec<(usize, String, VariableKind)> = Vec::new(); - let mut defined: Vec = Vec::new(); - for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) { - let cell = h.trim(); - if cell.is_empty() { - return Err(bad(C, format!("empty header in column {}", i + 1))); - } - let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?; - columns.push((i, name, kind)); - } - // Lock Variables refuses New Variable, so the import is refused when a header would define one. - if !defined.is_empty() { - variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?; - } - if rows.len() == 1 { - return Err(bad(C, "CSV has a header but no data rows")); - } - let mut parsed: Vec = Vec::new(); - // The names of the rows read so far, and the last number given to a row without a name. - let (mut seen, mut number) = (BTreeSet::new(), 0usize); - for &(line, ref row) in rows.iter().skip(1) { - let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" }; - let name = if raw_name.is_empty() { - // Same numbering as Capture Data Set when a row leaves the name blank or the file has - // no name column. The search starts after the last number given. - let (n, name) = (number + 1..) - .map(|n| (n, format!("Data Set {n}"))) - .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n)) - .unwrap_or_else(|| (line, format!("Data Set {line}"))); - number = n; - name - } else if raw_name.chars().count() > MAX_NAME { - return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters"))); - } else { - raw_name.to_string() - }; - if !seen.insert(name.clone()) { - return Err(bad(C, format!("line {line}: dataset `{name}` appears twice"))); - } - let mut values = BTreeMap::new(); - for &(col, ref var, kind) in &columns { - let cell = row.get(col).map(String::as_str).unwrap_or(""); - match cell_to_value(kind, cell, var, C) { - Ok(Some(v)) => { - values.insert(var.clone(), v); - } - Ok(None) => {} - Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))), - } - } - parsed.push(DataSet { name, values }); - } - let names: Vec = parsed.iter().map(|d| d.name.clone()).collect(); - let new_variables: Vec = defined.iter().map(|v| v.name.clone()).collect(); - let (imported, updated) = s.edit("Import Data Sets", |d, _| { - d.variables.variables.extend(defined); - let mut imported = 0usize; - let mut updated = 0usize; - for row in parsed { - if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) { - *existing = row; - updated += 1; - } else { - d.variables.datasets.push(row); - imported += 1; - } - } - Ok((imported, updated)) - })?; - Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables})) -} - -/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`. -fn dataset_export(s: &mut Session, p: &Value) -> Result { - const C: &str = "dataset.export"; - let st = s.doc()?; - if st.doc.variables.variables.is_empty() { - return Err(bad(C, "define a variable first")); - } - let csv = datasets_to_csv(&st.doc.variables); - if csv.len() > MAX_CSV_BYTES { - return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes"))); - } - match str_param(p, "path") { - Some(path) => { - let path = absolute_path(path); - write_file(&path, csv.as_bytes())?; - Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()})) - } - None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})), - } -} - fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result { let st = s.doc()?; let n = st.doc.variables.datasets.len(); @@ -902,289 +552,6 @@ fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result { apply_dataset(s, &name) } -/// Bytes of a variable library from `path`, `xml` or `dataBase64`. -fn library_bytes(p: &Value, cmd: &str) -> Result> { - let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) { - // One byte past the cap is enough to refuse the file below. - (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?, - (None, Some(xml), _) => xml.as_bytes().to_vec(), - (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?, - _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")), - }; - if bytes.len() > MAX_LIBRARY_BYTES { - return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes"))); - } - Ok(bytes) -} - -/// `xml` without the document type declaration of its prolog, with each reference to a general -/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped, -/// outside CDATA sections, comments and processing instructions. The declaration may hold an -/// external identifier and an internal subset of white space, comments, processing instructions and -/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`] -/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is -/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses -/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the -/// elements the parser reads. -fn without_doctype(xml: &str, cmd: &str) -> Result { - let b = xml.as_bytes(); - let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len()); - let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count()); - let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s)); - let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text"); - // The prolog: an XML declaration, white space, comments and processing instructions. - let mut i = 0; - loop { - i = spaces(i); - if at(i, "").ok_or_else(refused)?; - } else if at(i, "").ok_or_else(refused)?; - } else { - break; - } - } - if !at(i, " Option<(usize, &str)> { - let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?; - let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?; - Some((i + len + 2, xml.get(i + 1..i + 1 + len)?)) - }; - // `' - i = spaces(i + "')).count()); - i = spaces(i); - for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] { - if at(i, id) { - i += id.len(); - for _ in 0..literals { - i = literal(spaces(i)).ok_or_else(refused)?.0; - } - i = spaces(i); - } - } - let mut entities: Vec<(&str, String)> = Vec::new(); - if at(i, "[") { - i += 1; - loop { - i = spaces(i); - if at(i, "]") { - i = spaces(i + 1); - break; - } else if at(i, "").ok_or_else(refused)?; - } else if at(i, "").ok_or_else(refused)?; - } else if at(i, "')).count()); - let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?; - let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?; - if value.contains(['<', '&', '%']) { - return Err(refused()); - } - i = spaces(end); - if !at(i, ">") { - return Err(refused()); - } - i += 1; - if entities.len() == MAX_LIBRARY_ENTITIES { - return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities"))); - } - entities.push((name, value.replace('"', """).replace('\'', "'").replace('>', ">"))); - } else { - return Err(refused()); - } - } - } - if !at(i, ">") { - return Err(refused()); - } - let mut out = String::with_capacity(xml.len()); - out.push_str(xml.get(..start).unwrap_or_default()); - // The rest, copied with the references replaced outside CDATA sections, comments and - // processing instructions, where the parser reads none. - let mut i = i + 1; - while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) { - let verbatim = [(""), (""), ("")].into_iter().find(|(open, _)| at(i, open)); - if let Some((open, close)) = verbatim { - let end = past(i + open.len(), close).unwrap_or(xml.len()); - out.push_str(xml.get(i..end).unwrap_or_default()); - i = end; - continue; - } - // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far. - let entity = if c == '&' { - let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?)); - name.and_then(|name| entities.iter().find(|(n, _)| *n == name)) - } else { - None - }; - match entity { - Some((name, value)) => { - out.push_str(value); - i += name.len() + 2; - if out.len() > MAX_LIBRARY_BYTES { - return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes"))); - } - } - None => { - out.push(c); - i += c.len_utf8(); - } - } - } - if out.len() > MAX_LIBRARY_BYTES { - return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes"))); - } - Ok(out) -} - -/// A variable library as Load Variable Library reads it: the names of its text variables in the -/// order it declares them, the name and `trait` of each variable of another kind, and its data -/// sets with their text values. -struct Library { - text: Vec, - other: Vec<(String, String)>, - datasets: Vec, -} - -/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each -/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets` -/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names. -/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named -/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and -/// none when it has no `p` element. Values for variables of other kinds are left out. A variable -/// or data set declared twice, a value for a variable the library doesn't declare, and a library -/// past the caps are refused. Entities that its document type declaration defines are read as -/// [`without_doctype`] says. -fn parse_library(xml: &str, cmd: &str) -> Result { - let xml = without_doctype(xml, cmd)?; - // roxmltree parses nested elements recursively. - if super::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING { - return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep"))); - } - let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() }; - let doc = - roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?; - let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name; - let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else { - return Err(bad(cmd, "the file holds no `variableSets` element")); - }; - let sets: Vec = root.children().filter(|n| is(n, "variableSet")).collect(); - let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new()); - for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) { - let name = named(var.attribute("varName"), "varName", cmd)?; - if !declared.insert(name.clone()) { - return Err(bad(cmd, format!("variable `{name}` is declared twice"))); - } - if declared.len() > MAX_LIBRARY_VARIABLES { - return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables"))); - } - match var.attribute("trait") { - Some("textcontent") => text.push(name), - Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())), - None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))), - } - } - if declared.is_empty() { - return Err(bad(cmd, "the variable library declares no variables")); - } - let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect(); - let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new()); - for row in - sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet")) - { - let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?; - if !seen.insert(name.clone()) { - return Err(bad(cmd, format!("data set `{name}` is declared twice"))); - } - if seen.len() > MAX_LIBRARY_DATASETS { - return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets"))); - } - let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new()); - for el in row.children().filter(roxmltree::Node::is_element) { - let var = el.tag_name().name(); - if !declared.contains(var) { - return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare"))); - } - if !held.insert(var) { - return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`"))); - } - if !text_names.contains(var) { - continue; - } - let paras: Vec = el - .children() - .filter(|n| is(n, "p")) - .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect()) - .collect(); - if !paras.is_empty() { - values.insert(var.to_string(), DataValue::Text(paras.join("\n"))); - } - } - datasets.push(DataSet { name, values }); - } - Ok(Library { text, other, datasets }) -} - -/// Load Variable Library: replace the document's variables and data sets with a variable library's -/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings -/// and values, and every data set is deleted. The library's text variables are then defined, -/// unbound, and its data sets are added. The library's variables of other kinds are left out, named -/// in `skipped` and in a warning. -fn load_library(s: &mut Session, p: &Value) -> Result { - const C: &str = "variable.loadLibrary"; - let bytes = library_bytes(p, C)?; - let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?; - let lib = parse_library(xml, C)?; - let vars = &s.doc()?.doc.variables; - let deleted: Vec = vars.variables.iter().map(|v| v.name.clone()).collect(); - let deleted_sets: Vec = vars.datasets.iter().map(|d| d.name.clone()).collect(); - // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would - // delete or define a variable. - if !deleted.is_empty() || !lib.text.is_empty() { - variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?; - } - let added: Vec = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect(); - let names: Vec = lib.datasets.iter().map(|d| d.name.clone()).collect(); - s.edit("Load Variable Library", |d, _| { - d.variables.prune(&deleted); - d.variables.variables.extend(added); - d.variables.datasets = lib.datasets; - d.variables.active_dataset = None; - Ok(()) - })?; - // A highlight of a variable that is gone highlights nothing. - let st = s.doc_mut()?; - if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) { - st.variables_highlight = None; - } - st.revision += 1; - let skipped: Vec = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect(); - let mut out = json!({ - "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped, - }); - // Each `trait` once, in the order the library first uses it, with how many variables have it. - let mut kinds: Vec<(&str, usize)> = Vec::new(); - for (_, t) in &lib.other { - match kinds.iter_mut().find(|(k, _)| k == t) { - Some((_, n)) => *n += 1, - None => kinds.push((t, 1)), - } - } - if !kinds.is_empty() { - let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::>().join(", "); - out["warning"] = json!(format!("variables that aren't text were left out: {kinds}")); - } - Ok(out) -} - pub fn specs() -> Vec { vec![ cmd!( diff --git a/crates/engine/src/cmd/variablecmds/csv.rs b/crates/engine/src/cmd/variablecmds/csv.rs new file mode 100644 index 000000000..6cb10284b --- /dev/null +++ b/crates/engine/src/cmd/variablecmds/csv.rs @@ -0,0 +1,351 @@ +use super::*; + +/// Cap on a graph variable's CSV (MCP and files are untrusted). +pub(super) const MAX_GRAPH_CSV: usize = 256 * 1024; + +/// Caps on a datasets CSV (paths and MCP payloads are untrusted). +const MAX_CSV_BYTES: usize = 1_048_576; + +const MAX_CSV_ROWS: usize = 1_000; + +const MAX_CSV_COLS: usize = 256; + +/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`. +fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result> { + let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) { + // One byte past the cap is enough to refuse the file below. + (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?, + (None, Some(text), ..) => text.as_bytes().to_vec(), + (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?, + (None, None, None, Some(csv)) => csv.as_bytes().to_vec(), + _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")), + }; + if bytes.len() > MAX_CSV_BYTES { + return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes"))); + } + Ok(bytes) +} + +/// A row of a datasets CSV: the file line it starts on, and its cells. +type CsvRow = (usize, Vec); + +/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that +/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside +/// quotes. +fn tab_separated(text: &str) -> bool { + // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds + // anything but white space yet. + let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false); + let mut chars = text.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '"' if in_quotes => { + if chars.peek() == Some(&'"') { + chars.next(); + blank = false; + } else { + in_quotes = false; + } + } + '"' if start => in_quotes = true, + c if in_quotes => blank &= c.is_whitespace(), + ',' => return false, + '\t' => { + tab = true; + start = true; + continue; + } + '\r' => continue, + '\n' if blank => { + tab = false; + start = true; + continue; + } + '\n' => return tab, + c => blank &= c.is_whitespace(), + } + start = false; + } + tab +} + +/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated +/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside), +/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted +/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and +/// drops white space after the closing quote at the end of the cell. Blank lines are skipped. +fn parse_datasets_csv(text: &str, cmd: &str) -> Result> { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let sep = if tab_separated(text) { '\t' } else { ',' }; + let (mut rows, mut row, mut cur) = (Vec::new(), Vec::::new(), String::new()); + // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the + // current row starts on. + let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize); + let mut chars = text.chars().peekable(); + loop { + let c = chars.next(); + match c { + Some('"') if in_quotes => { + if chars.peek() == Some(&'"') { + cur.push('"'); + chars.next(); + } else { + in_quotes = false; + closed = Some(cur.len()); + } + } + Some('"') if cur.is_empty() => in_quotes = true, + Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)), + Some('\r') if !in_quotes => {} + Some('\n') | None if !in_quotes => { + row.push(end_csv_cell(&mut cur, &mut closed)); + if row.iter().any(|c| !c.trim().is_empty()) { + rows.push((start, std::mem::take(&mut row))); + // +1 for the header. + if rows.len() > MAX_CSV_ROWS + 1 { + return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows"))); + } + } + row.clear(); + if c.is_none() { + return Ok(rows); + } + line += 1; + start = line; + } + None => return Err(bad(cmd, format!("line {line}: unclosed quote"))), + Some(c) => { + line += usize::from(c == '\n'); + cur.push(c); + } + } + if row.len() > MAX_CSV_COLS { + return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns"))); + } + } +} + +/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell +/// is trimmed; with one, the text before it is kept as written and the text after it is +/// trimmed at the end. +fn end_csv_cell(cur: &mut String, closed: &mut Option) -> String { + let cell = std::mem::take(cur); + match closed.take() { + Some(at) => match cell.split_at_checked(at) { + Some((quoted, after)) => format!("{quoted}{}", after.trim_end()), + None => cell, + }, + None => cell.trim().to_string(), + } +} + +fn csv_escape(s: &str) -> String { + if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() } +} + +/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive). +fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => Ok(true), + "false" | "0" | "no" => Ok(false), + _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))), + } +} + +fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result> { + // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and + // visibility cells are trimmed. + let t = if kind == VariableKind::Text { cell } else { cell.trim() }; + if t.is_empty() { + return Ok(None); + } + Ok(Some(match kind { + VariableKind::Text => DataValue::Text(t.to_string()), + VariableKind::Image => DataValue::Image(t.to_string()), + VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()), + VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))), + VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?), + })) +} + +fn value_to_cell(v: &DataValue) -> String { + match v { + DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(), + DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(), + } +} + +/// Build the CSV of every dataset (header `name` then variables in definition order). +fn datasets_to_csv(vars: &Variables) -> String { + let mut out = String::from("name"); + for v in &vars.variables { + out.push(','); + out.push_str(&csv_escape(&v.name)); + } + out.push('\n'); + for ds in &vars.datasets { + out.push_str(&csv_escape(&ds.name)); + for v in &vars.variables { + out.push(','); + if let Some(val) = ds.values.get(&v.name) { + out.push_str(&csv_escape(&value_to_cell(val))); + } + } + out.push('\n'); + } + out +} + +/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names, +/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables +/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a +/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text +/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new +/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused. +fn header_variable( + cell: &str, + column: usize, + kinds: &BTreeMap, + defined: &mut Vec, + cmd: &str, +) -> Result<(String, VariableKind)> { + let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind)); + if let Some(kind) = known(cell, defined) { + return Ok((cell.to_string(), kind)); + } + let (name, kind) = if let Some(rest) = cell.strip_prefix('#') { + (rest.trim(), VariableKind::Visibility) + } else if let Some(rest) = cell.strip_prefix('@') { + (rest.trim(), VariableKind::Image) + } else if cell.starts_with('%') { + return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read"))); + } else { + (cell, VariableKind::Text) + }; + if name.is_empty() { + return Err(bad(cmd, format!("column {column}: `{cell}` names no variable"))); + } + match known(name, defined) { + Some(k) if k == kind => Ok((name.to_string(), kind)), + Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))), + None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))), + None => { + defined.push(Variable { name: name.to_string(), kind }); + Ok((name.to_string(), kind)) + } + } +} + +/// Import datasets from CSV. A first header cell `name` makes the first column the dataset +/// names; the other headers name variables, as [`header_variable`] reads them, and the +/// variables the document doesn't define yet are defined, unbound. Without the `name` column, +/// every header names a variable and the rows take the first free names "Data Set 1", "Data +/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One +/// undo step. +pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result { + const C: &str = "dataset.import"; + let bytes = datasets_csv_bytes(p, C)?; + let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?; + let rows = parse_datasets_csv(text, C)?; + let Some((_, header)) = rows.first() else { + return Err(bad(C, "CSV is empty")); + }; + let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name")); + let (kinds, existing_names) = { + let st = s.doc()?; + let kinds: BTreeMap = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect(); + let existing: BTreeSet = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect(); + (kinds, existing) + }; + let mut columns: Vec<(usize, String, VariableKind)> = Vec::new(); + let mut defined: Vec = Vec::new(); + for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) { + let cell = h.trim(); + if cell.is_empty() { + return Err(bad(C, format!("empty header in column {}", i + 1))); + } + let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?; + columns.push((i, name, kind)); + } + // Lock Variables refuses New Variable, so the import is refused when a header would define one. + if !defined.is_empty() { + variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?; + } + if rows.len() == 1 { + return Err(bad(C, "CSV has a header but no data rows")); + } + let mut parsed: Vec = Vec::new(); + // The names of the rows read so far, and the last number given to a row without a name. + let (mut seen, mut number) = (BTreeSet::new(), 0usize); + for &(line, ref row) in rows.iter().skip(1) { + let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" }; + let name = if raw_name.is_empty() { + // Same numbering as Capture Data Set when a row leaves the name blank or the file has + // no name column. The search starts after the last number given. + let (n, name) = (number + 1..) + .map(|n| (n, format!("Data Set {n}"))) + .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n)) + .unwrap_or_else(|| (line, format!("Data Set {line}"))); + number = n; + name + } else if raw_name.chars().count() > MAX_NAME { + return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters"))); + } else { + raw_name.to_string() + }; + if !seen.insert(name.clone()) { + return Err(bad(C, format!("line {line}: dataset `{name}` appears twice"))); + } + let mut values = BTreeMap::new(); + for &(col, ref var, kind) in &columns { + let cell = row.get(col).map(String::as_str).unwrap_or(""); + match cell_to_value(kind, cell, var, C) { + Ok(Some(v)) => { + values.insert(var.clone(), v); + } + Ok(None) => {} + Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))), + } + } + parsed.push(DataSet { name, values }); + } + let names: Vec = parsed.iter().map(|d| d.name.clone()).collect(); + let new_variables: Vec = defined.iter().map(|v| v.name.clone()).collect(); + let (imported, updated) = s.edit("Import Data Sets", |d, _| { + d.variables.variables.extend(defined); + let mut imported = 0usize; + let mut updated = 0usize; + for row in parsed { + if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) { + *existing = row; + updated += 1; + } else { + d.variables.datasets.push(row); + imported += 1; + } + } + Ok((imported, updated)) + })?; + Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables})) +} + +/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`. +pub(super) fn dataset_export(s: &mut Session, p: &Value) -> Result { + const C: &str = "dataset.export"; + let st = s.doc()?; + if st.doc.variables.variables.is_empty() { + return Err(bad(C, "define a variable first")); + } + let csv = datasets_to_csv(&st.doc.variables); + if csv.len() > MAX_CSV_BYTES { + return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes"))); + } + match str_param(p, "path") { + Some(path) => { + let path = absolute_path(path); + write_file(&path, csv.as_bytes())?; + Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()})) + } + None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})), + } +} diff --git a/crates/engine/src/cmd/variablecmds/library.rs b/crates/engine/src/cmd/variablecmds/library.rs new file mode 100644 index 000000000..83374a16d --- /dev/null +++ b/crates/engine/src/cmd/variablecmds/library.rs @@ -0,0 +1,298 @@ +use super::*; + +/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how +/// deep its elements nest, and the entities, variables and data sets it declares. +const MAX_LIBRARY_BYTES: usize = 8 << 20; + +const MAX_LIBRARY_ENTITIES: usize = 64; + +const MAX_LIBRARY_NODES: u32 = 2_000_000; + +const MAX_LIBRARY_NESTING: usize = 256; + +const MAX_LIBRARY_VARIABLES: usize = 256; + +const MAX_LIBRARY_DATASETS: usize = 1_000; + +/// Bytes of a variable library from `path`, `xml` or `dataBase64`. +fn library_bytes(p: &Value, cmd: &str) -> Result> { + let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) { + // One byte past the cap is enough to refuse the file below. + (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?, + (None, Some(xml), _) => xml.as_bytes().to_vec(), + (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?, + _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")), + }; + if bytes.len() > MAX_LIBRARY_BYTES { + return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes"))); + } + Ok(bytes) +} + +/// `xml` without the document type declaration of its prolog, with each reference to a general +/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped, +/// outside CDATA sections, comments and processing instructions. The declaration may hold an +/// external identifier and an internal subset of white space, comments, processing instructions and +/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`] +/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is +/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses +/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the +/// elements the parser reads. +fn without_doctype(xml: &str, cmd: &str) -> Result { + let b = xml.as_bytes(); + let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len()); + let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count()); + let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s)); + let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text"); + // The prolog: an XML declaration, white space, comments and processing instructions. + let mut i = 0; + loop { + i = spaces(i); + if at(i, "").ok_or_else(refused)?; + } else if at(i, "").ok_or_else(refused)?; + } else { + break; + } + } + if !at(i, " Option<(usize, &str)> { + let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?; + let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?; + Some((i + len + 2, xml.get(i + 1..i + 1 + len)?)) + }; + // `' + i = spaces(i + "')).count()); + i = spaces(i); + for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] { + if at(i, id) { + i += id.len(); + for _ in 0..literals { + i = literal(spaces(i)).ok_or_else(refused)?.0; + } + i = spaces(i); + } + } + let mut entities: Vec<(&str, String)> = Vec::new(); + if at(i, "[") { + i += 1; + loop { + i = spaces(i); + if at(i, "]") { + i = spaces(i + 1); + break; + } else if at(i, "").ok_or_else(refused)?; + } else if at(i, "").ok_or_else(refused)?; + } else if at(i, "')).count()); + let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?; + let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?; + if value.contains(['<', '&', '%']) { + return Err(refused()); + } + i = spaces(end); + if !at(i, ">") { + return Err(refused()); + } + i += 1; + if entities.len() == MAX_LIBRARY_ENTITIES { + return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities"))); + } + entities.push((name, value.replace('"', """).replace('\'', "'").replace('>', ">"))); + } else { + return Err(refused()); + } + } + } + if !at(i, ">") { + return Err(refused()); + } + let mut out = String::with_capacity(xml.len()); + out.push_str(xml.get(..start).unwrap_or_default()); + // The rest, copied with the references replaced outside CDATA sections, comments and + // processing instructions, where the parser reads none. + let mut i = i + 1; + while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) { + let verbatim = [(""), (""), ("")].into_iter().find(|(open, _)| at(i, open)); + if let Some((open, close)) = verbatim { + let end = past(i + open.len(), close).unwrap_or(xml.len()); + out.push_str(xml.get(i..end).unwrap_or_default()); + i = end; + continue; + } + // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far. + let entity = if c == '&' { + let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?)); + name.and_then(|name| entities.iter().find(|(n, _)| *n == name)) + } else { + None + }; + match entity { + Some((name, value)) => { + out.push_str(value); + i += name.len() + 2; + if out.len() > MAX_LIBRARY_BYTES { + return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes"))); + } + } + None => { + out.push(c); + i += c.len_utf8(); + } + } + } + if out.len() > MAX_LIBRARY_BYTES { + return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes"))); + } + Ok(out) +} + +/// A variable library as Load Variable Library reads it: the names of its text variables in the +/// order it declares them, the name and `trait` of each variable of another kind, and its data +/// sets with their text values. +struct Library { + text: Vec, + other: Vec<(String, String)>, + datasets: Vec, +} + +/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each +/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets` +/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names. +/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named +/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and +/// none when it has no `p` element. Values for variables of other kinds are left out. A variable +/// or data set declared twice, a value for a variable the library doesn't declare, and a library +/// past the caps are refused. Entities that its document type declaration defines are read as +/// [`without_doctype`] says. +fn parse_library(xml: &str, cmd: &str) -> Result { + let xml = without_doctype(xml, cmd)?; + // roxmltree parses nested elements recursively. + if crate::cmd::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING { + return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep"))); + } + let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() }; + let doc = + roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?; + let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name; + let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else { + return Err(bad(cmd, "the file holds no `variableSets` element")); + }; + let sets: Vec = root.children().filter(|n| is(n, "variableSet")).collect(); + let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new()); + for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) { + let name = named(var.attribute("varName"), "varName", cmd)?; + if !declared.insert(name.clone()) { + return Err(bad(cmd, format!("variable `{name}` is declared twice"))); + } + if declared.len() > MAX_LIBRARY_VARIABLES { + return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables"))); + } + match var.attribute("trait") { + Some("textcontent") => text.push(name), + Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())), + None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))), + } + } + if declared.is_empty() { + return Err(bad(cmd, "the variable library declares no variables")); + } + let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect(); + let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new()); + for row in + sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet")) + { + let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?; + if !seen.insert(name.clone()) { + return Err(bad(cmd, format!("data set `{name}` is declared twice"))); + } + if seen.len() > MAX_LIBRARY_DATASETS { + return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets"))); + } + let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new()); + for el in row.children().filter(roxmltree::Node::is_element) { + let var = el.tag_name().name(); + if !declared.contains(var) { + return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare"))); + } + if !held.insert(var) { + return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`"))); + } + if !text_names.contains(var) { + continue; + } + let paras: Vec = el + .children() + .filter(|n| is(n, "p")) + .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect()) + .collect(); + if !paras.is_empty() { + values.insert(var.to_string(), DataValue::Text(paras.join("\n"))); + } + } + datasets.push(DataSet { name, values }); + } + Ok(Library { text, other, datasets }) +} + +/// Load Variable Library: replace the document's variables and data sets with a variable library's +/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings +/// and values, and every data set is deleted. The library's text variables are then defined, +/// unbound, and its data sets are added. The library's variables of other kinds are left out, named +/// in `skipped` and in a warning. +pub(super) fn load_library(s: &mut Session, p: &Value) -> Result { + const C: &str = "variable.loadLibrary"; + let bytes = library_bytes(p, C)?; + let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?; + let lib = parse_library(xml, C)?; + let vars = &s.doc()?.doc.variables; + let deleted: Vec = vars.variables.iter().map(|v| v.name.clone()).collect(); + let deleted_sets: Vec = vars.datasets.iter().map(|d| d.name.clone()).collect(); + // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would + // delete or define a variable. + if !deleted.is_empty() || !lib.text.is_empty() { + variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?; + } + let added: Vec = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect(); + let names: Vec = lib.datasets.iter().map(|d| d.name.clone()).collect(); + s.edit("Load Variable Library", |d, _| { + d.variables.prune(&deleted); + d.variables.variables.extend(added); + d.variables.datasets = lib.datasets; + d.variables.active_dataset = None; + Ok(()) + })?; + // A highlight of a variable that is gone highlights nothing. + let st = s.doc_mut()?; + if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) { + st.variables_highlight = None; + } + st.revision += 1; + let skipped: Vec = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect(); + let mut out = json!({ + "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped, + }); + // Each `trait` once, in the order the library first uses it, with how many variables have it. + let mut kinds: Vec<(&str, usize)> = Vec::new(); + for (_, t) in &lib.other { + match kinds.iter_mut().find(|(k, _)| k == t) { + Some((_, n)) => *n += 1, + None => kinds.push((t, 1)), + } + } + if !kinds.is_empty() { + let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::>().join(", "); + out["warning"] = json!(format!("variables that aren't text were left out: {kinds}")); + } + Ok(out) +}