Compare commits

...
Author SHA1 Message Date
Dylan M. Taylor 97ee78977e fix(mailings): collapse delimiter counter guard 2026-11-15 10:15:08 -05:00
Dylan M. Taylor 3bf5520721 Mailings: bound delimiter storage and JSON serialization before allocation 2026-11-15 09:30:35 -05:00
Dylan M. Taylor 7221277ebe Mailings: refuse over-budget normalized recipients before padding or cloning 2026-11-15 09:28:33 -05:00
Mat Strange 9ac8dd94e1 Mailings: import bounded CSV parsing from #4055
Source: 0f84914edc. Preserve current multiline delimiter detection; integration follows separately.
2026-11-15 09:20:18 -05:00
3 changed files with 236 additions and 37 deletions
+230 -36
View File
@@ -4,7 +4,7 @@ use super::{CmdError, CmdResult, Session, Value, json, p};
/// The field separator: whichever of `,` `;` and tab the first non-blank record (outside quotes) uses most.
fn csv_separator(text: &str) -> char {
let mut quoted = false;
let mut unquoted = String::new();
let mut counts = [0usize; 3];
// A quoted header may span physical lines; infer from the first complete record.
for c in text.split_inclusive(['\n', '\r']).skip_while(|line| line.trim().is_empty()).flat_map(str::chars) {
if c == '"' {
@@ -13,28 +13,42 @@ fn csv_separator(text: &str) -> char {
if c == '\n' || c == '\r' {
break;
}
unquoted.push(c);
if let Some(i) = [',', ';', '\t'].iter().position(|candidate| *candidate == c)
&& let Some(count) = counts.get_mut(i)
{
*count = count.saturating_add(1);
}
}
}
[',', ';', '\t'].into_iter().max_by_key(|c| unquoted.matches(*c).count()).unwrap_or(',')
[',', ';', '\t'].into_iter().zip(counts).max_by_key(|(_, count)| *count).map_or(',', |(c, _)| c)
}
/// Ends the current row: kept when it has any non-empty field, otherwise dropped.
fn push_csv_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>) {
if row.iter().any(|x| !x.is_empty()) {
rows.push(std::mem::take(row));
} else {
row.clear();
}
/// What a CSV may hold while it is read: the parser refuses a list that is over these, so a
/// hostile file is never built and then cut.
#[derive(Clone, Copy)]
pub struct CsvLimits {
/// Most columns in a record.
pub fields: usize,
/// Most cells in the list: its records times its widest record.
pub cells: usize,
/// Most bytes of cell text in all.
pub text: usize,
}
/// Parse CSV (RFC 4180-ish: quotes, doubled quotes, commas/semicolons/tabs).
pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
///
/// The limits are kept while the list is read, not after it: a record wider than
/// `limits.fields`, a list of more than `limits.cells` cells, or more than `limits.text` bytes
/// of cell text is refused, rather than built and then cut.
fn parse_csv_limited(text: &str, limits: &CsvLimits) -> Result<Vec<Vec<String>>, CmdError> {
let sep = csv_separator(text);
let mut rows = Vec::new();
let mut row = Vec::new();
let mut cell = String::new();
let mut q = false;
// The widest record read so far, and the bytes of cell text read so far.
let mut widest = 0usize;
let mut text_bytes = 0usize;
let mut chars = text.chars().peekable();
while let Some(c) = chars.next() {
if q {
@@ -51,29 +65,86 @@ pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
} else if c == '"' {
q = true;
} else if c == sep {
row.push(std::mem::take(&mut cell));
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
} else if c == '\n' || c == '\r' {
if c == '\r' && chars.peek() == Some(&'\n') {
chars.next();
}
row.push(std::mem::take(&mut cell));
push_csv_row(&mut rows, &mut row);
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
end_row(&mut rows, &mut row, &mut widest, limits)?;
} else {
cell.push(c);
}
if rows.len() > 100_000 {
break;
// The text read so far: the finished cells and the one being read.
if text_bytes.saturating_add(cell.len()) > limits.text {
return Err(CmdError::Failed(format!("the CSV can't have more than {} of text", text_size(limits.text))));
}
if rows.len() > MAX_RECORDS + 1 {
return Err(CmdError::Failed("the CSV has too many records".into()));
}
}
row.push(cell);
push_csv_row(&mut rows, &mut row);
rows
// The last cell, when the file doesn't end with a line break.
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
end_row(&mut rows, &mut row, &mut widest, limits)?;
Ok(rows)
}
/// A finished cell joins the record: the record's width stays inside the limit.
fn end_cell(row: &mut Vec<String>, cell: &mut String, text_bytes: &mut usize, limits: &CsvLimits) -> Result<(), CmdError> {
*text_bytes = text_bytes.saturating_add(cell.len());
row.push(std::mem::take(cell));
if row.len() > limits.fields {
return Err(CmdError::Failed(format!("the CSV can't have more than {} columns", limits.fields)));
}
Ok(())
}
/// A finished record joins the list (an empty one is dropped). The list costs its records times
/// its widest record — a recipient list pads every record to the header's width — so a list
/// over the limit is refused here, while it is read.
fn end_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>, widest: &mut usize, limits: &CsvLimits) -> Result<(), CmdError> {
if row.iter().any(|x| !x.is_empty()) {
*widest = (*widest).max(row.len());
rows.push(std::mem::take(row));
} else {
row.clear();
}
if rows.len().saturating_mul(*widest) > limits.cells {
return Err(CmdError::Failed(format!("the CSV can't have more than {} cells (its records times its widest record)", limits.cells)));
}
Ok(())
}
/// A byte budget, said the way a reader says it.
fn text_size(bytes: usize) -> String {
if bytes >= 1 << 20 { format!("{} MB", bytes >> 20) } else { format!("{bytes} bytes") }
}
/// Most records and fields a recipient list keeps.
const MAX_RECORDS: usize = 100_000;
const MAX_FIELDS: usize = 1_000;
const MAX_CELLS: usize = 2_000_000;
const MAX_TEXT_BYTES: usize = 64 << 20;
const RECIPIENT_LIMITS: CsvLimits = CsvLimits { fields: MAX_FIELDS, cells: MAX_CELLS, text: MAX_TEXT_BYTES };
/// Refuse excessive CSV input instead of returning a silently truncated list.
pub fn parse_csv(text: &str) -> Result<Vec<Vec<String>>, CmdError> {
parse_csv_limited(text, &RECIPIENT_LIMITS)
}
// A sparse list is padded and cloned: charge its normalized shape BEFORE allocating it.
fn check_shape(records: usize, fields: usize, max_cells: usize) -> Result<(), CmdError> {
if records > MAX_RECORDS || fields > MAX_FIELDS || records.checked_mul(fields).is_none_or(|n| n > max_cells) {
return Err(CmdError::Failed("the recipient list exceeds its record, field or aggregate cell limit".into()));
}
Ok(())
}
fn spend_text(left: &mut usize, n: usize) -> Result<(), CmdError> {
*left = left.checked_sub(n).ok_or_else(|| CmdError::Failed("the recipient list exceeds its text budget".into()))?;
Ok(())
}
pub(super) fn cell(x: &Value) -> String {
match x {
@@ -83,11 +154,54 @@ pub(super) fn cell(x: &Value) -> String {
}
}
// Serialize non-string JSON cells through a bounded sink, not to_string() before charging.
fn budget_cell(value: &Value, left: &mut usize) -> Result<String, CmdError> {
match value {
Value::Null => Ok(String::new()),
Value::String(text) => {
spend_text(left, text.len())?;
Ok(text.clone())
}
_ => {
struct Sink {
bytes: Vec<u8>,
limit: usize,
}
impl std::io::Write for Sink {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
return Err(std::io::Error::other("recipient text budget"));
}
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut sink = Sink { bytes: Vec::new(), limit: *left };
serde_json::to_writer(&mut sink, value).map_err(|_| CmdError::Failed("the recipient list exceeds its text budget".into()))?;
spend_text(left, sink.bytes.len())?;
String::from_utf8(sink.bytes).map_err(|_| CmdError::Failed("invalid recipient text".into()))
}
}
}
/// A recipient table from `rows` (objects keyed by field, or arrays in `fields` order). Without
/// `fields`, the columns are the objects' keys.
fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<String>, Vec<Vec<String>>), CmdError> {
let mut left = MAX_TEXT_BYTES;
let headers: Vec<String> = match fields {
Some(f) => f.iter().take(MAX_FIELDS).map(|h| cell(h).trim().to_string()).collect(),
Some(f) => {
check_shape(0, f.len(), MAX_CELLS)?;
let mut headers = Vec::new();
for h in f {
let text = budget_cell(h, &mut left)?.trim().to_string();
headers.push(text);
}
headers
}
None => {
if arr.iter().any(Value::is_array) {
return Err(CmdError::Params("`fields` is required when `rows` are arrays".into()));
@@ -95,7 +209,9 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
let mut headers: Vec<String> = Vec::new();
for o in arr.iter().filter_map(Value::as_object) {
for k in o.keys() {
if !headers.contains(k) && headers.len() < MAX_FIELDS {
if !headers.contains(k) {
check_shape(0, headers.len().saturating_add(1), MAX_CELLS)?;
spend_text(&mut left, k.len())?;
headers.push(k.clone());
}
}
@@ -103,21 +219,34 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
headers
}
};
let rows = arr
.iter()
.take(MAX_RECORDS)
.filter_map(|r| match r {
Value::Array(cells) => Some((0..headers.len()).map(|i| cells.get(i).map(cell).unwrap_or_default()).collect()),
Value::Object(o) => Some(headers.iter().map(|h| o.get(h).map(cell).unwrap_or_default()).collect()),
_ => None,
})
.collect();
let count = arr.iter().filter(|v| v.is_array() || v.is_object()).count();
check_shape(count, headers.len(), MAX_CELLS)?;
let mut rows = Vec::new();
for value in arr {
let mut row = Vec::new();
if !value.is_array() && !value.is_object() {
continue;
}
for (i, h) in headers.iter().enumerate() {
let v = match value {
Value::Array(a) => a.get(i),
Value::Object(o) => o.get(h),
_ => None,
};
let text = match v {
Some(v) => budget_cell(v, &mut left)?,
None => String::new(),
};
row.push(text);
}
rows.push(row);
}
Ok((headers, rows))
}
/// Skip unquoted blank prefix lines only when a text recipient list has a header.
/// Headerless lists and the shared CSV reader keep whitespace records.
fn recipient_csv(text: &str, headers: bool) -> Vec<Vec<String>> {
fn recipient_csv(text: &str, headers: bool) -> Result<Vec<Vec<String>>, CmdError> {
let prefix = text
.split_inclusive(['\n', '\r'])
.take_while(|line| headers && line.trim().is_empty())
@@ -131,7 +260,7 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
table_from_rows(v.get("fields").and_then(Value::as_array), arr)?
} else {
let all = if let Some(c) = p::str(v, "csv") {
recipient_csv(c, p::bool(v, "headers").unwrap_or(true))
recipient_csv(c, p::bool(v, "headers").unwrap_or(true))?
} else {
let (bytes, ext) = if let Some(d) = p::str(v, "data") {
let b = crate::cmd::insert::base64_decode(d.trim()).ok_or_else(|| CmdError::Params("`data` isn't valid base64".into()))?;
@@ -188,7 +317,7 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
sheet = json!(chosen);
rows
} else {
recipient_csv(&decode(&bytes, v)?, p::bool(v, "headers").unwrap_or(true))
recipient_csv(&decode(&bytes, v)?, p::bool(v, "headers").unwrap_or(true))?
}
};
header_split(all, p::bool(v, "headers").unwrap_or(true))?
@@ -243,14 +372,79 @@ fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>
}
let fields = if headers {
let first = all.remove(0);
wordcraft_formats::sheet::field_names(&first[..first.len().min(MAX_FIELDS)])
check_shape(0, first.len(), MAX_CELLS)?;
wordcraft_formats::sheet::field_names(&first)
} else {
let width = all.iter().map(Vec::len).max().unwrap_or(0).min(MAX_FIELDS);
let width = all.iter().map(Vec::len).max().unwrap_or(0);
check_shape(0, width, MAX_CELLS)?;
(1..=width).map(|i| format!("Column {i}")).collect()
};
all.truncate(MAX_RECORDS);
check_shape(all.len(), fields.len(), MAX_CELLS)?;
let mut left = MAX_TEXT_BYTES;
for h in &fields {
spend_text(&mut left, h.len())?;
}
for row in &all {
for text in row.iter().take(fields.len()) {
spend_text(&mut left, text.len())?;
}
}
for r in &mut all {
r.resize(fields.len(), String::new());
}
Ok((fields, all))
}
#[cfg(test)]
mod budget_tests {
use super::*;
#[test]
fn sparse_rows_are_charged_at_their_padded_width() {
let limits = CsvLimits { fields: 3, cells: 8, text: 100 };
assert!(parse_csv_limited("a,b,c\nx\nx\n", &limits).is_err());
assert!(check_shape(3, 3, 8).is_err());
assert!(check_shape(2, 3, 6).is_ok());
assert!(check_shape(usize::MAX, usize::MAX, usize::MAX).is_err());
}
#[test]
fn width_text_and_record_limits_refuse_instead_of_truncating() {
let limits = CsvLimits { fields: 2, cells: 20, text: 4 };
assert!(parse_csv_limited("a,b,c", &limits).is_err());
assert!(parse_csv_limited("abcde", &limits).is_err());
assert!(parse_csv_limited("a\nb\n", &limits).is_ok());
assert!(parse_csv(&"x\n".repeat(MAX_RECORDS + 2)).is_err());
}
#[test]
fn rejected_csv_and_json_keep_the_existing_list() {
let mut session = Session::new(wordcraft_doc::Document::from_text("Hello"));
recipients(&mut session, &json!({"csv": "Name\nKeep"})).unwrap();
let fields: Vec<String> = (0..MAX_FIELDS).map(|i| format!("F{i}")).collect();
let count = MAX_CELLS / MAX_FIELDS + 1;
let csv = format!("{}\n{}", fields.join(","), "x\n".repeat(count));
assert!(recipients(&mut session, &json!({"csv": csv})).is_err());
let rows: Vec<Value> = (0..count).map(|_| json!({})).collect();
assert!(recipients(&mut session, &json!({"fields": fields, "rows": rows})).is_err());
assert_eq!(session.merge.headers, vec!["Name"]);
assert_eq!(session.merge.rows, vec![vec!["Keep"]]);
assert_eq!(session.merge.source, session.merge.rows);
}
#[test]
fn complex_json_text_is_bounded_during_serialization() {
let value = json!({"nested": ["abcdefgh", {"tail": true}]});
let mut left = 5;
assert!(budget_cell(&value, &mut left).is_err());
let mut left = 100;
let text = budget_cell(&value, &mut left).unwrap();
assert_eq!(text, value.to_string());
assert_eq!(left, 100 - text.len());
}
#[test]
fn delimiter_counting_preserves_ties_and_multiline_quotes() {
assert_eq!(csv_separator("A,B;C\n1,2;3"), ';');
assert_eq!(csv_separator("A;B\tC\n1;2\t3"), '\t');
assert_eq!(csv_separator("\n\"A,\nB\";C\nx;y"), ';');
let limits = CsvLimits { fields: 2, cells: 8, text: 4 };
assert!(parse_csv_limited(&"x".repeat(100_000), &limits).is_err());
}
}
+5
View File
@@ -1,5 +1,10 @@
use super::*;
// Keep existing delimiter/parity assertions exercising the now-fallible public parser.
fn parse_csv(text: &str) -> Vec<Vec<String>> {
super::parse_csv(text).unwrap()
}
#[test]
fn csv_parsing() {
let r = parse_csv("Name,City\n\"Smith, Ann\",Paris\r\nBo,\"Oslo \"\"N\"\"\"\n\n");
+1 -1
View File
@@ -348,7 +348,7 @@ pub(super) fn customize_ribbon(s: &mut Session, v: &Value) -> CmdResult {
pub(super) fn spreadsheet_rows(v: &Value) -> Result<Vec<Vec<String>>, CmdError> {
// The CSV fallback is for an absent `values`; an explicit null is a malformed argument.
let Some(values) = v.get("values") else {
return Ok(crate::cmd::mailings::parse_csv(p::str(v, "csv").unwrap_or("A,B,C\n1,2,3\n4,5,6")));
return crate::cmd::mailings::parse_csv(p::str(v, "csv").unwrap_or("A,B,C\n1,2,3\n4,5,6"));
};
let list = values.as_array().ok_or_else(|| CmdError::Params("`values` must be an array of rows".into()))?;
if list.is_empty() || list.len() > 1000 {