mirror of
https://github.com/storytold/wordcraft.git
synced 2026-11-18 03:44:59 -06:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97ee78977e | ||
|
|
3bf5520721 | ||
|
|
7221277ebe | ||
|
|
9ac8dd94e1 |
@@ -4,7 +4,7 @@ use super::{CmdError, CmdResult, Session, Value, json, p};
|
||||
/// The field separator: whichever of `,` `;` and tab the first non-blank record (outside quotes) uses most.
|
||||
fn csv_separator(text: &str) -> char {
|
||||
let mut quoted = false;
|
||||
let mut unquoted = String::new();
|
||||
let mut counts = [0usize; 3];
|
||||
// A quoted header may span physical lines; infer from the first complete record.
|
||||
for c in text.split_inclusive(['\n', '\r']).skip_while(|line| line.trim().is_empty()).flat_map(str::chars) {
|
||||
if c == '"' {
|
||||
@@ -13,28 +13,42 @@ fn csv_separator(text: &str) -> char {
|
||||
if c == '\n' || c == '\r' {
|
||||
break;
|
||||
}
|
||||
unquoted.push(c);
|
||||
if let Some(i) = [',', ';', '\t'].iter().position(|candidate| *candidate == c)
|
||||
&& let Some(count) = counts.get_mut(i)
|
||||
{
|
||||
*count = count.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
[',', ';', '\t'].into_iter().max_by_key(|c| unquoted.matches(*c).count()).unwrap_or(',')
|
||||
[',', ';', '\t'].into_iter().zip(counts).max_by_key(|(_, count)| *count).map_or(',', |(c, _)| c)
|
||||
}
|
||||
|
||||
/// Ends the current row: kept when it has any non-empty field, otherwise dropped.
|
||||
fn push_csv_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>) {
|
||||
if row.iter().any(|x| !x.is_empty()) {
|
||||
rows.push(std::mem::take(row));
|
||||
} else {
|
||||
row.clear();
|
||||
}
|
||||
/// What a CSV may hold while it is read: the parser refuses a list that is over these, so a
|
||||
/// hostile file is never built and then cut.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct CsvLimits {
|
||||
/// Most columns in a record.
|
||||
pub fields: usize,
|
||||
/// Most cells in the list: its records times its widest record.
|
||||
pub cells: usize,
|
||||
/// Most bytes of cell text in all.
|
||||
pub text: usize,
|
||||
}
|
||||
|
||||
/// Parse CSV (RFC 4180-ish: quotes, doubled quotes, commas/semicolons/tabs).
|
||||
pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
|
||||
///
|
||||
/// The limits are kept while the list is read, not after it: a record wider than
|
||||
/// `limits.fields`, a list of more than `limits.cells` cells, or more than `limits.text` bytes
|
||||
/// of cell text is refused, rather than built and then cut.
|
||||
fn parse_csv_limited(text: &str, limits: &CsvLimits) -> Result<Vec<Vec<String>>, CmdError> {
|
||||
let sep = csv_separator(text);
|
||||
let mut rows = Vec::new();
|
||||
let mut row = Vec::new();
|
||||
let mut cell = String::new();
|
||||
let mut q = false;
|
||||
// The widest record read so far, and the bytes of cell text read so far.
|
||||
let mut widest = 0usize;
|
||||
let mut text_bytes = 0usize;
|
||||
let mut chars = text.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if q {
|
||||
@@ -51,29 +65,86 @@ pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
|
||||
} else if c == '"' {
|
||||
q = true;
|
||||
} else if c == sep {
|
||||
row.push(std::mem::take(&mut cell));
|
||||
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
|
||||
} else if c == '\n' || c == '\r' {
|
||||
if c == '\r' && chars.peek() == Some(&'\n') {
|
||||
chars.next();
|
||||
}
|
||||
row.push(std::mem::take(&mut cell));
|
||||
push_csv_row(&mut rows, &mut row);
|
||||
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
|
||||
end_row(&mut rows, &mut row, &mut widest, limits)?;
|
||||
} else {
|
||||
cell.push(c);
|
||||
}
|
||||
if rows.len() > 100_000 {
|
||||
break;
|
||||
// The text read so far: the finished cells and the one being read.
|
||||
if text_bytes.saturating_add(cell.len()) > limits.text {
|
||||
return Err(CmdError::Failed(format!("the CSV can't have more than {} of text", text_size(limits.text))));
|
||||
}
|
||||
if rows.len() > MAX_RECORDS + 1 {
|
||||
return Err(CmdError::Failed("the CSV has too many records".into()));
|
||||
}
|
||||
}
|
||||
row.push(cell);
|
||||
push_csv_row(&mut rows, &mut row);
|
||||
rows
|
||||
// The last cell, when the file doesn't end with a line break.
|
||||
end_cell(&mut row, &mut cell, &mut text_bytes, limits)?;
|
||||
end_row(&mut rows, &mut row, &mut widest, limits)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// A finished cell joins the record: the record's width stays inside the limit.
|
||||
fn end_cell(row: &mut Vec<String>, cell: &mut String, text_bytes: &mut usize, limits: &CsvLimits) -> Result<(), CmdError> {
|
||||
*text_bytes = text_bytes.saturating_add(cell.len());
|
||||
row.push(std::mem::take(cell));
|
||||
if row.len() > limits.fields {
|
||||
return Err(CmdError::Failed(format!("the CSV can't have more than {} columns", limits.fields)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A finished record joins the list (an empty one is dropped). The list costs its records times
|
||||
/// its widest record — a recipient list pads every record to the header's width — so a list
|
||||
/// over the limit is refused here, while it is read.
|
||||
fn end_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>, widest: &mut usize, limits: &CsvLimits) -> Result<(), CmdError> {
|
||||
if row.iter().any(|x| !x.is_empty()) {
|
||||
*widest = (*widest).max(row.len());
|
||||
rows.push(std::mem::take(row));
|
||||
} else {
|
||||
row.clear();
|
||||
}
|
||||
if rows.len().saturating_mul(*widest) > limits.cells {
|
||||
return Err(CmdError::Failed(format!("the CSV can't have more than {} cells (its records times its widest record)", limits.cells)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A byte budget, said the way a reader says it.
|
||||
fn text_size(bytes: usize) -> String {
|
||||
if bytes >= 1 << 20 { format!("{} MB", bytes >> 20) } else { format!("{bytes} bytes") }
|
||||
}
|
||||
|
||||
/// Most records and fields a recipient list keeps.
|
||||
const MAX_RECORDS: usize = 100_000;
|
||||
|
||||
const MAX_FIELDS: usize = 1_000;
|
||||
const MAX_CELLS: usize = 2_000_000;
|
||||
const MAX_TEXT_BYTES: usize = 64 << 20;
|
||||
const RECIPIENT_LIMITS: CsvLimits = CsvLimits { fields: MAX_FIELDS, cells: MAX_CELLS, text: MAX_TEXT_BYTES };
|
||||
|
||||
/// Refuse excessive CSV input instead of returning a silently truncated list.
|
||||
pub fn parse_csv(text: &str) -> Result<Vec<Vec<String>>, CmdError> {
|
||||
parse_csv_limited(text, &RECIPIENT_LIMITS)
|
||||
}
|
||||
|
||||
// A sparse list is padded and cloned: charge its normalized shape BEFORE allocating it.
|
||||
fn check_shape(records: usize, fields: usize, max_cells: usize) -> Result<(), CmdError> {
|
||||
if records > MAX_RECORDS || fields > MAX_FIELDS || records.checked_mul(fields).is_none_or(|n| n > max_cells) {
|
||||
return Err(CmdError::Failed("the recipient list exceeds its record, field or aggregate cell limit".into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn spend_text(left: &mut usize, n: usize) -> Result<(), CmdError> {
|
||||
*left = left.checked_sub(n).ok_or_else(|| CmdError::Failed("the recipient list exceeds its text budget".into()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn cell(x: &Value) -> String {
|
||||
match x {
|
||||
@@ -83,11 +154,54 @@ pub(super) fn cell(x: &Value) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize non-string JSON cells through a bounded sink, not to_string() before charging.
|
||||
fn budget_cell(value: &Value, left: &mut usize) -> Result<String, CmdError> {
|
||||
match value {
|
||||
Value::Null => Ok(String::new()),
|
||||
Value::String(text) => {
|
||||
spend_text(left, text.len())?;
|
||||
Ok(text.clone())
|
||||
}
|
||||
_ => {
|
||||
struct Sink {
|
||||
bytes: Vec<u8>,
|
||||
limit: usize,
|
||||
}
|
||||
impl std::io::Write for Sink {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("recipient text budget"));
|
||||
}
|
||||
self.bytes.try_reserve(bytes.len()).map_err(std::io::Error::other)?;
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut sink = Sink { bytes: Vec::new(), limit: *left };
|
||||
serde_json::to_writer(&mut sink, value).map_err(|_| CmdError::Failed("the recipient list exceeds its text budget".into()))?;
|
||||
spend_text(left, sink.bytes.len())?;
|
||||
String::from_utf8(sink.bytes).map_err(|_| CmdError::Failed("invalid recipient text".into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A recipient table from `rows` (objects keyed by field, or arrays in `fields` order). Without
|
||||
/// `fields`, the columns are the objects' keys.
|
||||
fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<String>, Vec<Vec<String>>), CmdError> {
|
||||
let mut left = MAX_TEXT_BYTES;
|
||||
let headers: Vec<String> = match fields {
|
||||
Some(f) => f.iter().take(MAX_FIELDS).map(|h| cell(h).trim().to_string()).collect(),
|
||||
Some(f) => {
|
||||
check_shape(0, f.len(), MAX_CELLS)?;
|
||||
let mut headers = Vec::new();
|
||||
for h in f {
|
||||
let text = budget_cell(h, &mut left)?.trim().to_string();
|
||||
headers.push(text);
|
||||
}
|
||||
headers
|
||||
}
|
||||
None => {
|
||||
if arr.iter().any(Value::is_array) {
|
||||
return Err(CmdError::Params("`fields` is required when `rows` are arrays".into()));
|
||||
@@ -95,7 +209,9 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
|
||||
let mut headers: Vec<String> = Vec::new();
|
||||
for o in arr.iter().filter_map(Value::as_object) {
|
||||
for k in o.keys() {
|
||||
if !headers.contains(k) && headers.len() < MAX_FIELDS {
|
||||
if !headers.contains(k) {
|
||||
check_shape(0, headers.len().saturating_add(1), MAX_CELLS)?;
|
||||
spend_text(&mut left, k.len())?;
|
||||
headers.push(k.clone());
|
||||
}
|
||||
}
|
||||
@@ -103,21 +219,34 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
|
||||
headers
|
||||
}
|
||||
};
|
||||
let rows = arr
|
||||
.iter()
|
||||
.take(MAX_RECORDS)
|
||||
.filter_map(|r| match r {
|
||||
Value::Array(cells) => Some((0..headers.len()).map(|i| cells.get(i).map(cell).unwrap_or_default()).collect()),
|
||||
Value::Object(o) => Some(headers.iter().map(|h| o.get(h).map(cell).unwrap_or_default()).collect()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let count = arr.iter().filter(|v| v.is_array() || v.is_object()).count();
|
||||
check_shape(count, headers.len(), MAX_CELLS)?;
|
||||
let mut rows = Vec::new();
|
||||
for value in arr {
|
||||
let mut row = Vec::new();
|
||||
if !value.is_array() && !value.is_object() {
|
||||
continue;
|
||||
}
|
||||
for (i, h) in headers.iter().enumerate() {
|
||||
let v = match value {
|
||||
Value::Array(a) => a.get(i),
|
||||
Value::Object(o) => o.get(h),
|
||||
_ => None,
|
||||
};
|
||||
let text = match v {
|
||||
Some(v) => budget_cell(v, &mut left)?,
|
||||
None => String::new(),
|
||||
};
|
||||
row.push(text);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
Ok((headers, rows))
|
||||
}
|
||||
|
||||
/// Skip unquoted blank prefix lines only when a text recipient list has a header.
|
||||
/// Headerless lists and the shared CSV reader keep whitespace records.
|
||||
fn recipient_csv(text: &str, headers: bool) -> Vec<Vec<String>> {
|
||||
fn recipient_csv(text: &str, headers: bool) -> Result<Vec<Vec<String>>, CmdError> {
|
||||
let prefix = text
|
||||
.split_inclusive(['\n', '\r'])
|
||||
.take_while(|line| headers && line.trim().is_empty())
|
||||
@@ -131,7 +260,7 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
|
||||
table_from_rows(v.get("fields").and_then(Value::as_array), arr)?
|
||||
} else {
|
||||
let all = if let Some(c) = p::str(v, "csv") {
|
||||
recipient_csv(c, p::bool(v, "headers").unwrap_or(true))
|
||||
recipient_csv(c, p::bool(v, "headers").unwrap_or(true))?
|
||||
} else {
|
||||
let (bytes, ext) = if let Some(d) = p::str(v, "data") {
|
||||
let b = crate::cmd::insert::base64_decode(d.trim()).ok_or_else(|| CmdError::Params("`data` isn't valid base64".into()))?;
|
||||
@@ -188,7 +317,7 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
|
||||
sheet = json!(chosen);
|
||||
rows
|
||||
} else {
|
||||
recipient_csv(&decode(&bytes, v)?, p::bool(v, "headers").unwrap_or(true))
|
||||
recipient_csv(&decode(&bytes, v)?, p::bool(v, "headers").unwrap_or(true))?
|
||||
}
|
||||
};
|
||||
header_split(all, p::bool(v, "headers").unwrap_or(true))?
|
||||
@@ -243,14 +372,79 @@ fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>
|
||||
}
|
||||
let fields = if headers {
|
||||
let first = all.remove(0);
|
||||
wordcraft_formats::sheet::field_names(&first[..first.len().min(MAX_FIELDS)])
|
||||
check_shape(0, first.len(), MAX_CELLS)?;
|
||||
wordcraft_formats::sheet::field_names(&first)
|
||||
} else {
|
||||
let width = all.iter().map(Vec::len).max().unwrap_or(0).min(MAX_FIELDS);
|
||||
let width = all.iter().map(Vec::len).max().unwrap_or(0);
|
||||
check_shape(0, width, MAX_CELLS)?;
|
||||
(1..=width).map(|i| format!("Column {i}")).collect()
|
||||
};
|
||||
all.truncate(MAX_RECORDS);
|
||||
check_shape(all.len(), fields.len(), MAX_CELLS)?;
|
||||
let mut left = MAX_TEXT_BYTES;
|
||||
for h in &fields {
|
||||
spend_text(&mut left, h.len())?;
|
||||
}
|
||||
for row in &all {
|
||||
for text in row.iter().take(fields.len()) {
|
||||
spend_text(&mut left, text.len())?;
|
||||
}
|
||||
}
|
||||
for r in &mut all {
|
||||
r.resize(fields.len(), String::new());
|
||||
}
|
||||
Ok((fields, all))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod budget_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn sparse_rows_are_charged_at_their_padded_width() {
|
||||
let limits = CsvLimits { fields: 3, cells: 8, text: 100 };
|
||||
assert!(parse_csv_limited("a,b,c\nx\nx\n", &limits).is_err());
|
||||
assert!(check_shape(3, 3, 8).is_err());
|
||||
assert!(check_shape(2, 3, 6).is_ok());
|
||||
assert!(check_shape(usize::MAX, usize::MAX, usize::MAX).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn width_text_and_record_limits_refuse_instead_of_truncating() {
|
||||
let limits = CsvLimits { fields: 2, cells: 20, text: 4 };
|
||||
assert!(parse_csv_limited("a,b,c", &limits).is_err());
|
||||
assert!(parse_csv_limited("abcde", &limits).is_err());
|
||||
assert!(parse_csv_limited("a\nb\n", &limits).is_ok());
|
||||
assert!(parse_csv(&"x\n".repeat(MAX_RECORDS + 2)).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn rejected_csv_and_json_keep_the_existing_list() {
|
||||
let mut session = Session::new(wordcraft_doc::Document::from_text("Hello"));
|
||||
recipients(&mut session, &json!({"csv": "Name\nKeep"})).unwrap();
|
||||
let fields: Vec<String> = (0..MAX_FIELDS).map(|i| format!("F{i}")).collect();
|
||||
let count = MAX_CELLS / MAX_FIELDS + 1;
|
||||
let csv = format!("{}\n{}", fields.join(","), "x\n".repeat(count));
|
||||
assert!(recipients(&mut session, &json!({"csv": csv})).is_err());
|
||||
let rows: Vec<Value> = (0..count).map(|_| json!({})).collect();
|
||||
assert!(recipients(&mut session, &json!({"fields": fields, "rows": rows})).is_err());
|
||||
assert_eq!(session.merge.headers, vec!["Name"]);
|
||||
assert_eq!(session.merge.rows, vec![vec!["Keep"]]);
|
||||
assert_eq!(session.merge.source, session.merge.rows);
|
||||
}
|
||||
#[test]
|
||||
fn complex_json_text_is_bounded_during_serialization() {
|
||||
let value = json!({"nested": ["abcdefgh", {"tail": true}]});
|
||||
let mut left = 5;
|
||||
assert!(budget_cell(&value, &mut left).is_err());
|
||||
let mut left = 100;
|
||||
let text = budget_cell(&value, &mut left).unwrap();
|
||||
assert_eq!(text, value.to_string());
|
||||
assert_eq!(left, 100 - text.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimiter_counting_preserves_ties_and_multiline_quotes() {
|
||||
assert_eq!(csv_separator("A,B;C\n1,2;3"), ';');
|
||||
assert_eq!(csv_separator("A;B\tC\n1;2\t3"), '\t');
|
||||
assert_eq!(csv_separator("\n\"A,\nB\";C\nx;y"), ';');
|
||||
let limits = CsvLimits { fields: 2, cells: 8, text: 4 };
|
||||
assert!(parse_csv_limited(&"x".repeat(100_000), &limits).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
use super::*;
|
||||
|
||||
// Keep existing delimiter/parity assertions exercising the now-fallible public parser.
|
||||
fn parse_csv(text: &str) -> Vec<Vec<String>> {
|
||||
super::parse_csv(text).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_parsing() {
|
||||
let r = parse_csv("Name,City\n\"Smith, Ann\",Paris\r\nBo,\"Oslo \"\"N\"\"\"\n\n");
|
||||
|
||||
@@ -348,7 +348,7 @@ pub(super) fn customize_ribbon(s: &mut Session, v: &Value) -> CmdResult {
|
||||
pub(super) fn spreadsheet_rows(v: &Value) -> Result<Vec<Vec<String>>, CmdError> {
|
||||
// The CSV fallback is for an absent `values`; an explicit null is a malformed argument.
|
||||
let Some(values) = v.get("values") else {
|
||||
return Ok(crate::cmd::mailings::parse_csv(p::str(v, "csv").unwrap_or("A,B,C\n1,2,3\n4,5,6")));
|
||||
return crate::cmd::mailings::parse_csv(p::str(v, "csv").unwrap_or("A,B,C\n1,2,3\n4,5,6"));
|
||||
};
|
||||
let list = values.as_array().ok_or_else(|| CmdError::Params("`values` must be an array of rows".into()))?;
|
||||
if list.is_empty() || list.len() > 1000 {
|
||||
|
||||
Reference in New Issue
Block a user