mirror of
https://github.com/storytold/wordcraft.git
synced 2026-11-11 16:31:54 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
03153bf241 | ||
|
|
8087582ee2 | ||
|
|
2b8ed17d9c | ||
|
|
a064d1caf8 | ||
|
|
f02e3afbdf |
@@ -25,8 +25,8 @@ use merge::{address_map, finish, go_to, merge_fields, step};
|
||||
pub use merge::{merge_range, refresh};
|
||||
#[cfg(test)]
|
||||
use merge::{mergefield_name, placeholder};
|
||||
use recipients::recipients;
|
||||
pub use recipients::{RECIPIENT_EXTENSIONS, is_spreadsheet_name, parse_csv};
|
||||
pub use recipients::{CsvCap, RECIPIENT_EXTENSIONS, is_spreadsheet_name, parse_csv, parse_csv_streaming};
|
||||
use recipients::{recipients, recipients_strict};
|
||||
|
||||
/// The mail-merge data source and preview state (lives in the session, not the document).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
@@ -79,7 +79,8 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
// Edited records keep the list's filter and sort (rules on removed fields drop).
|
||||
// Copies, so a list that fails to load leaves the filter and sort as they were.
|
||||
let (filter, sort) = (s.merge.filter.clone(), s.merge.sort.clone());
|
||||
let r = recipients(s, &json!({"rows": v.get("rows"), "fields": v.get("fields")}))?;
|
||||
// An edit that would lose records is refused, not truncated.
|
||||
let r = recipients_strict(s, &json!({"rows": v.get("rows"), "fields": v.get("fields")}))?;
|
||||
s.merge.filter = filter;
|
||||
s.merge.sort = sort;
|
||||
apply_filter(s);
|
||||
|
||||
@@ -27,11 +27,52 @@ fn push_csv_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>) {
|
||||
}
|
||||
|
||||
pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
|
||||
parse_csv_streaming(text, None).0
|
||||
}
|
||||
|
||||
/// Cap for [`parse_csv_streaming`]: rows are kept only while the cells kept stay within `max_cells`
|
||||
/// (and the rows within [`MAX_RECORDS`], plus the header).
|
||||
/// With `width_from_first_row`, every kept row is cut to the first row's width (a header row's
|
||||
/// fields are all a list keeps), otherwise to [`MAX_FIELDS`].
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct CsvCap {
|
||||
pub max_cells: usize,
|
||||
pub width_from_first_row: bool,
|
||||
}
|
||||
|
||||
/// [`parse_csv`] with an optional streaming cap (off by default): once the kept rows hold
|
||||
/// `max_cells`, later rows are counted but not kept, so peak memory is the cap plus one row, not
|
||||
/// every row in the file. Returns the rows and how many records the cap left out. Without a cap
|
||||
/// it parses at most 100,000 rows, as before.
|
||||
pub fn parse_csv_streaming(text: &str, cap: Option<CsvCap>) -> (Vec<Vec<String>>, usize) {
|
||||
let sep = csv_separator(text);
|
||||
let mut rows = Vec::new();
|
||||
let mut row = Vec::new();
|
||||
let mut cell = String::new();
|
||||
let mut q = false;
|
||||
let (mut kept_cells, mut omitted) = (0usize, 0usize);
|
||||
// Ends a row: kept unless the cap is spent.
|
||||
let mut end_row = |rows: &mut Vec<Vec<String>>, row: &mut Vec<String>| {
|
||||
if !row.iter().any(|x| !x.is_empty()) {
|
||||
row.clear();
|
||||
return;
|
||||
}
|
||||
let Some(cap) = cap else { return push_csv_row(rows, row) };
|
||||
let width = if cap.width_from_first_row { rows.first().map_or(MAX_FIELDS, Vec::len) } else { MAX_FIELDS };
|
||||
row.truncate(width.max(1));
|
||||
// A header row is not a record: it takes no part of the cell budget (the rows path keeps
|
||||
// `max_records(fields)` records beside its header too).
|
||||
let free = cap.width_from_first_row && rows.is_empty();
|
||||
if rows.len() > MAX_RECORDS || (!free && kept_cells.saturating_add(row.len()) > cap.max_cells) {
|
||||
omitted += 1;
|
||||
row.clear();
|
||||
} else {
|
||||
if !free {
|
||||
kept_cells += row.len();
|
||||
}
|
||||
rows.push(std::mem::take(row));
|
||||
}
|
||||
};
|
||||
let mut chars = text.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if q {
|
||||
@@ -54,17 +95,17 @@ pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
|
||||
chars.next();
|
||||
}
|
||||
row.push(std::mem::take(&mut cell));
|
||||
push_csv_row(&mut rows, &mut row);
|
||||
end_row(&mut rows, &mut row);
|
||||
} else {
|
||||
cell.push(c);
|
||||
}
|
||||
if rows.len() > 100_000 {
|
||||
if cap.is_none() && rows.len() > 100_000 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
row.push(cell);
|
||||
push_csv_row(&mut rows, &mut row);
|
||||
rows
|
||||
end_row(&mut rows, &mut row);
|
||||
(rows, omitted)
|
||||
}
|
||||
|
||||
/// Most records and fields a recipient list keeps.
|
||||
@@ -72,6 +113,36 @@ const MAX_RECORDS: usize = 100_000;
|
||||
|
||||
const MAX_FIELDS: usize = 1_000;
|
||||
|
||||
/// Most cells (records × fields) a recipient list keeps. Every record is padded to every field
|
||||
/// and the list is kept twice (as read, and filtered and sorted), so a short file with a wide
|
||||
/// header and many one-field rows would otherwise claim gigabytes of empty cells.
|
||||
const MAX_CELLS: usize = 4_000_000;
|
||||
|
||||
/// Most records a list with `fields` fields keeps: [`MAX_RECORDS`], fewer for a wide list.
|
||||
fn max_records(fields: usize) -> usize {
|
||||
MAX_RECORDS.min(MAX_CELLS / fields.max(1))
|
||||
}
|
||||
|
||||
/// Cut `rows` to what a list with `fields` fields keeps, saying so when it drops any. Returns how
|
||||
/// many records it dropped.
|
||||
fn cap_records<T>(rows: &mut Vec<T>, fields: usize) -> usize {
|
||||
let keep = max_records(fields);
|
||||
let dropped = rows.len().saturating_sub(keep);
|
||||
if dropped > 0 {
|
||||
log::warn!("the recipient list keeps at most {keep} records with {fields} fields; the rest are left out");
|
||||
rows.truncate(keep);
|
||||
}
|
||||
dropped
|
||||
}
|
||||
|
||||
/// The error for a list that would lose records: `keep` is what `fields` fields allow.
|
||||
fn over_limit(dropped: usize, fields: usize) -> CmdError {
|
||||
CmdError::Params(format!(
|
||||
"the recipient list has {dropped} more records than the {} that fit with {fields} fields; remove records or fields",
|
||||
max_records(fields)
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn cell(x: &Value) -> String {
|
||||
match x {
|
||||
Value::Null => String::new(),
|
||||
@@ -82,7 +153,7 @@ pub(super) fn cell(x: &Value) -> String {
|
||||
|
||||
/// A recipient table from `rows` (objects keyed by field, or arrays in `fields` order). Without
|
||||
/// `fields`, the columns are the objects' keys.
|
||||
fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<String>, Vec<Vec<String>>), CmdError> {
|
||||
fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<String>, Vec<Vec<String>>, usize), CmdError> {
|
||||
let headers: Vec<String> = match fields {
|
||||
Some(f) => f.iter().take(MAX_FIELDS).map(|h| cell(h).trim().to_string()).collect(),
|
||||
None => {
|
||||
@@ -100,25 +171,45 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
|
||||
headers
|
||||
}
|
||||
};
|
||||
// Only arrays and objects are records; a scalar entry takes no slot under the cap.
|
||||
let valid = |r: &&Value| r.is_array() || r.is_object();
|
||||
let keep = max_records(headers.len());
|
||||
let rows = arr
|
||||
.iter()
|
||||
.take(MAX_RECORDS)
|
||||
.filter_map(|r| match r {
|
||||
Value::Array(cells) => Some((0..headers.len()).map(|i| cells.get(i).map(cell).unwrap_or_default()).collect()),
|
||||
Value::Object(o) => Some(headers.iter().map(|h| o.get(h).map(cell).unwrap_or_default()).collect()),
|
||||
_ => None,
|
||||
.filter(valid)
|
||||
.take(keep)
|
||||
.map(|r| match r {
|
||||
Value::Array(cells) => (0..headers.len()).map(|i| cells.get(i).map(cell).unwrap_or_default()).collect(),
|
||||
Value::Object(o) => headers.iter().map(|h| o.get(h).map(cell).unwrap_or_default()).collect(),
|
||||
_ => Vec::new(),
|
||||
})
|
||||
.collect();
|
||||
Ok((headers, rows))
|
||||
let dropped = arr.iter().filter(valid).count().saturating_sub(keep);
|
||||
Ok((headers, rows, dropped))
|
||||
}
|
||||
|
||||
/// Loads a recipient list; records past the cap are left out and reported as `dropped`.
|
||||
pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
|
||||
load_recipients(s, v, false)
|
||||
}
|
||||
|
||||
/// Like [`recipients`], but a list that would lose records is refused and the current list kept.
|
||||
pub(super) fn recipients_strict(s: &mut Session, v: &Value) -> CmdResult {
|
||||
load_recipients(s, v, true)
|
||||
}
|
||||
|
||||
fn load_recipients(s: &mut Session, v: &Value, strict: bool) -> CmdResult {
|
||||
let mut sheet = Value::Null;
|
||||
let (headers, rows) = if let Some(arr) = v.get("rows").and_then(Value::as_array) {
|
||||
let (headers, rows, dropped) = if let Some(arr) = v.get("rows").and_then(Value::as_array) {
|
||||
table_from_rows(v.get("fields").and_then(Value::as_array), arr)?
|
||||
} else {
|
||||
let headers = p::bool(v, "headers").unwrap_or(true);
|
||||
let cap = Some(CsvCap { max_cells: MAX_CELLS, width_from_first_row: headers });
|
||||
let mut omitted = 0;
|
||||
let all = if let Some(c) = p::str(v, "csv") {
|
||||
parse_csv(c)
|
||||
let (rows, o) = parse_csv_streaming(c, cap);
|
||||
omitted = o;
|
||||
rows
|
||||
} else {
|
||||
let (bytes, ext) = if let Some(d) = p::str(v, "data") {
|
||||
let b = crate::cmd::insert::base64_decode(d.trim()).ok_or_else(|| CmdError::Params("`data` isn't valid base64".into()))?;
|
||||
@@ -175,11 +266,17 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
|
||||
sheet = json!(chosen);
|
||||
rows
|
||||
} else {
|
||||
parse_csv(&decode(&bytes, v)?)
|
||||
let (rows, o) = parse_csv_streaming(&decode(&bytes, v)?, cap);
|
||||
omitted = o;
|
||||
rows
|
||||
}
|
||||
};
|
||||
header_split(all, p::bool(v, "headers").unwrap_or(true))?
|
||||
let (fields, rows, dropped) = header_split(all, headers)?;
|
||||
(fields, rows, dropped + omitted)
|
||||
};
|
||||
if strict && dropped > 0 {
|
||||
return Err(over_limit(dropped, headers.len()));
|
||||
}
|
||||
s.merge.headers = headers;
|
||||
s.merge.source = rows.clone();
|
||||
s.merge.rows = rows;
|
||||
@@ -188,6 +285,9 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
|
||||
s.merge.record = 0;
|
||||
refresh_shown(s);
|
||||
let mut r = json!({"fields": s.merge.headers, "records": s.merge.rows.len()});
|
||||
if dropped > 0 {
|
||||
r["dropped"] = json!(dropped);
|
||||
}
|
||||
if !sheet.is_null() {
|
||||
r["sheet"] = sheet;
|
||||
}
|
||||
@@ -223,8 +323,9 @@ fn read_file(path: &str) -> Result<Vec<u8>, CmdError> {
|
||||
|
||||
/// Field names and records from a table's rows: with `headers`, the first row names the fields
|
||||
/// (blank names become `Column N`, repeats are numbered); otherwise every row is a record and
|
||||
/// the fields are `Column 1`…`Column N`. Records are padded or cut to the fields, and capped.
|
||||
fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>, Vec<Vec<String>>), CmdError> {
|
||||
/// the fields are `Column 1`…`Column N`. Records are padded or cut to the fields, and capped
|
||||
/// ([`max_records`]).
|
||||
fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>, Vec<Vec<String>>, usize), CmdError> {
|
||||
if all.is_empty() {
|
||||
return Err(CmdError::Failed("the recipient list is empty".into()));
|
||||
}
|
||||
@@ -235,9 +336,9 @@ fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>
|
||||
let width = all.iter().map(Vec::len).max().unwrap_or(0).min(MAX_FIELDS);
|
||||
(1..=width).map(|i| format!("Column {i}")).collect()
|
||||
};
|
||||
all.truncate(MAX_RECORDS);
|
||||
let dropped = cap_records(&mut all, fields.len());
|
||||
for r in &mut all {
|
||||
r.resize(fields.len(), String::new());
|
||||
}
|
||||
Ok((fields, all))
|
||||
Ok((fields, all, dropped))
|
||||
}
|
||||
|
||||
@@ -695,3 +695,84 @@ fn merge_uses_only_filtered_records_in_sort_order() {
|
||||
assert!(s.run("mailings.finish", &json!({})).is_err());
|
||||
assert!(s.run("mailings.editRecipients", &json!({})).is_ok());
|
||||
}
|
||||
|
||||
/// A wide header and many one-field rows: every record is padded to every field (and the list
|
||||
/// kept twice), so the cells the list keeps are capped, not just its records and fields.
|
||||
#[test]
|
||||
fn a_wide_sparse_recipient_list_is_capped_by_cells() {
|
||||
let header = (1..=1_000).map(|i| format!("F{i}")).collect::<Vec<_>>().join(",");
|
||||
let csv = format!("{header}\n{}", "x\n".repeat(20_000));
|
||||
let mut s = Session::new(Document::from_text("Hello"));
|
||||
let r = s.run("mailings.recipients", &json!({"csv": csv})).unwrap();
|
||||
assert_eq!(r["records"], 4_000);
|
||||
let rows: Vec<Value> = (0..20_000).map(|_| json!({})).collect();
|
||||
let fields: Vec<String> = (1..=1_000).map(|i| format!("F{i}")).collect();
|
||||
let r = s.run("mailings.recipients", &json!({"rows": rows, "fields": fields})).unwrap();
|
||||
assert_eq!(r["records"], 4_000);
|
||||
// A narrow list keeps its usual number of records.
|
||||
let r = s.run("mailings.recipients", &json!({"csv": format!("Name\n{}", "Ann\n".repeat(20_000))})).unwrap();
|
||||
assert_eq!(r["records"], 20_000);
|
||||
}
|
||||
|
||||
/// An import says how many records the cap left out; an edit that would lose records is refused
|
||||
/// and the list stays as it was.
|
||||
#[test]
|
||||
fn recipient_cap_is_reported_and_edits_that_exceed_it_are_refused() {
|
||||
let mut s = Session::new(Document::from_text("Hello"));
|
||||
let fields: Vec<String> = (1..=80).map(|i| format!("F{i}")).collect();
|
||||
let rows: Vec<Value> = (0..50_000).map(|_| json!({})).collect();
|
||||
let r = s.run("mailings.recipients", &json!({"rows": rows, "fields": fields})).unwrap();
|
||||
assert_eq!((r["records"].clone(), r.get("dropped")), (json!(50_000), None));
|
||||
let fields: Vec<String> = (1..=81).map(|i| format!("F{i}")).collect();
|
||||
let rows: Vec<Value> = (0..50_000).map(|_| json!({})).collect();
|
||||
let e = s.run("mailings.editRecipients", &json!({"rows": rows, "fields": fields.clone()}));
|
||||
assert!(matches!(e, Err(CmdError::Params(_))), "{e:?}");
|
||||
assert_eq!((s.merge.headers.len(), s.merge.rows.len()), (80, 50_000));
|
||||
let rows: Vec<Value> = (0..50_000).map(|_| json!({})).collect();
|
||||
let r = s.run("mailings.recipients", &json!({"rows": rows, "fields": fields})).unwrap();
|
||||
assert_eq!((r["records"].clone(), r["dropped"].clone()), (json!(49_382), json!(618)));
|
||||
}
|
||||
|
||||
/// A scalar entry is not a record, so it takes no slot under the cap.
|
||||
#[test]
|
||||
fn a_scalar_row_does_not_take_a_record_slot() {
|
||||
let mut s = Session::new(Document::from_text("Hello"));
|
||||
let fields: Vec<String> = (1..=80).map(|i| format!("F{i}")).collect();
|
||||
let mut rows: Vec<Value> = vec![json!(1)];
|
||||
rows.extend((0..50_000).map(|_| json!({})));
|
||||
let r = s.run("mailings.recipients", &json!({"rows": rows, "fields": fields})).unwrap();
|
||||
assert_eq!((r["records"].clone(), r.get("dropped")), (json!(50_000), None));
|
||||
}
|
||||
|
||||
/// The cell cap applies while the CSV is parsed: rows past it are counted, not kept, and a
|
||||
/// parse without a cap is unchanged.
|
||||
#[test]
|
||||
fn csv_streaming_cap_keeps_at_most_the_cells_and_counts_the_rest() {
|
||||
use super::{CsvCap, parse_csv_streaming};
|
||||
let csv = format!("a,b\n{}", "1,2\n".repeat(10));
|
||||
let cap = Some(CsvCap { max_cells: 8, width_from_first_row: true });
|
||||
let (rows, omitted) = parse_csv_streaming(&csv, cap);
|
||||
assert_eq!((rows.len(), omitted), (5, 6)); // the header is free; four records fit in 8 cells
|
||||
// Rows wider than the header are cut to it before they are counted.
|
||||
let (rows, omitted) = parse_csv_streaming("a,b\n1,2,3\n4,5,6\n", Some(CsvCap { max_cells: 2, width_from_first_row: true }));
|
||||
assert_eq!((rows, omitted), (vec![vec!["a".to_string(), "b".to_string()], vec!["1".to_string(), "2".to_string()]], 1));
|
||||
assert_eq!(parse_csv_streaming(&csv, None), (parse_csv(&csv), 0));
|
||||
// A full-width header takes none of the budget: exactly `max_cells / width` records are kept.
|
||||
let (rows, omitted) = parse_csv_streaming("a,b,c\n1,2,3\n4,5,6\n7,8,9\n", Some(CsvCap { max_cells: 6, width_from_first_row: true }));
|
||||
assert_eq!((rows.len(), omitted), (3, 1));
|
||||
// Without a header the first row is a record and is charged.
|
||||
let (rows, omitted) = parse_csv_streaming("1,2,3\n4,5,6\n7,8,9\n", Some(CsvCap { max_cells: 6, width_from_first_row: false }));
|
||||
assert_eq!((rows.len(), omitted), (2, 1));
|
||||
}
|
||||
|
||||
/// A CSV with a full-width header keeps as many records as the `rows` path does: the header takes
|
||||
/// no part of the cell budget.
|
||||
#[test]
|
||||
fn a_full_width_csv_header_does_not_cost_a_record() {
|
||||
let header = (1..=80).map(|i| format!("F{i}")).collect::<Vec<_>>().join(",");
|
||||
let record = format!("x{}", ",".repeat(79));
|
||||
let csv = format!("{header}\n{}", format!("{record}\n").repeat(50_000));
|
||||
let mut s = Session::new(Document::from_text("Hello"));
|
||||
let r = s.run("mailings.recipients", &json!({"csv": csv})).unwrap();
|
||||
assert_eq!((r["records"].clone(), r.get("dropped")), (json!(50_000), None));
|
||||
}
|
||||
|
||||
@@ -154,7 +154,6 @@ pub fn decode_font_file(bytes: &[u8]) -> Result<Vec<u8>, UserFontError> {
|
||||
unwrap_web_font(|| wuff::decompress_woff1(bytes))?
|
||||
}
|
||||
FontFormat::Woff2 => {
|
||||
check_woff2_collection(bytes)?;
|
||||
let mut too_many_points = false;
|
||||
let unwrapped = unwrap_web_font(|| {
|
||||
wuff::decompress_woff2_with_custom_brotli(bytes, &mut |data, size| {
|
||||
@@ -222,29 +221,8 @@ fn woff2_points_fit(file: &[u8], stream: &[u8]) -> bool {
|
||||
/// The transformed `glyf` tables in `stream`, found from the WOFF2 table directory in `file`
|
||||
/// (tables follow each other in directory order).
|
||||
fn woff2_transformed_glyf<'a>(file: &[u8], stream: &'a [u8]) -> Option<Vec<&'a [u8]>> {
|
||||
let (tables, _) = woff2_directory(file)?;
|
||||
let (mut offset, mut glyf) = (0usize, Vec::new());
|
||||
for t in tables {
|
||||
let end = offset.checked_add(t.len)?;
|
||||
if t.transformed_glyf {
|
||||
glyf.push(stream.get(offset..end)?);
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
Some(glyf)
|
||||
}
|
||||
|
||||
/// A WOFF2 table directory entry: whether it is a transformed `glyf` table, and its length in
|
||||
/// the decompressed stream.
|
||||
struct Woff2Table {
|
||||
transformed_glyf: bool,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
/// The WOFF2 table directory in `file`, and where it ends (the collection directory follows).
|
||||
fn woff2_directory(file: &[u8]) -> Option<(Vec<Woff2Table>, usize)> {
|
||||
let tables = u16::from_be_bytes(file.get(12..14)?.try_into().ok()?);
|
||||
let (mut at, mut out) = (48usize, Vec::with_capacity(usize::from(tables)));
|
||||
let (mut at, mut offset, mut glyf) = (48usize, 0usize, Vec::new());
|
||||
for _ in 0..tables {
|
||||
let flags = *file.get(at)?;
|
||||
at += 1;
|
||||
@@ -260,81 +238,13 @@ fn woff2_directory(file: &[u8]) -> Option<(Vec<Woff2Table>, usize)> {
|
||||
// glyf and loca are transformed at version 0, other tables at any other version.
|
||||
let transformed = if is_glyf || is_loca { flags >> 6 == 0 } else { flags >> 6 != 0 };
|
||||
let len = if transformed { base128(file, &mut at)? } else { orig };
|
||||
out.push(Woff2Table { transformed_glyf: is_glyf && transformed, len });
|
||||
}
|
||||
Some((out, at))
|
||||
}
|
||||
|
||||
/// Largest table directories a WOFF2 collection may rebuild, in bytes. Each font's directory is
|
||||
/// 16 bytes per table it lists, while the file spends as little as one byte per listed table, so
|
||||
/// a collection listing the same tables over and over rebuilt hundreds of megabytes of
|
||||
/// directories (twice: built, then copied into the font) before the decoder's output cap was
|
||||
/// checked. Real collections need kilobytes.
|
||||
const MAX_WOFF2_COLLECTION_HEADER: usize = 16 << 20;
|
||||
|
||||
/// Why a WOFF2 collection whose directories are too large, or list a table twice, is refused.
|
||||
const BAD_COLLECTION: &str = "the collection's font directories are too large or list a table twice";
|
||||
|
||||
/// WOFF2 collections: the table directories the decoder rebuilds for its fonts must fit
|
||||
/// [`MAX_WOFF2_COLLECTION_HEADER`], and no font may list a table twice (OpenType allows one
|
||||
/// table per tag in a font). Checked before decoding, since the decoder builds them before it
|
||||
/// checks its output size. A directory that doesn't parse is left to the decoder to report.
|
||||
fn check_woff2_collection(file: &[u8]) -> Result<(), UserFontError> {
|
||||
if file.get(4..8) != Some(b"ttcf".as_slice()) {
|
||||
return Ok(());
|
||||
}
|
||||
match woff2_collection_fits(file) {
|
||||
Some(false) => Err(UserFontError::BadWebFont(BAD_COLLECTION.into())),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Do the collection's rebuilt table directories fit, with no table listed twice in one font?
|
||||
/// None when the directories don't parse.
|
||||
fn woff2_collection_fits(file: &[u8]) -> Option<bool> {
|
||||
let (tables, mut at) = woff2_directory(file)?;
|
||||
let version = u32::from_be_bytes(file.get(at..at.checked_add(4)?)?.try_into().ok()?);
|
||||
at += 4;
|
||||
let fonts = read_255_u16(file, &mut at)?;
|
||||
// TTC header (with the DSIG fields of version 2) and one offset per font.
|
||||
let dsig = if version == 0x0002_0000 { 12 } else { 0 };
|
||||
let mut size = 12 + dsig + 4 * fonts;
|
||||
let mut indices: Vec<usize> = Vec::new();
|
||||
for _ in 0..fonts {
|
||||
let n = read_255_u16(file, &mut at)?;
|
||||
size = size.saturating_add(12 + 16 * n);
|
||||
if size > MAX_WOFF2_COLLECTION_HEADER {
|
||||
return Some(false);
|
||||
}
|
||||
at = at.checked_add(4)?; // flavor
|
||||
indices.clear();
|
||||
for _ in 0..n {
|
||||
let i = read_255_u16(file, &mut at)?;
|
||||
if i >= tables.len() {
|
||||
return None;
|
||||
}
|
||||
indices.push(i);
|
||||
}
|
||||
indices.sort_unstable();
|
||||
if indices.windows(2).any(|w| w[0] == w[1]) {
|
||||
return Some(false);
|
||||
let end = offset.checked_add(len)?;
|
||||
if is_glyf && transformed {
|
||||
glyf.push(stream.get(offset..end)?);
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
Some(true)
|
||||
}
|
||||
|
||||
/// WOFF2's 255UInt16.
|
||||
fn read_255_u16(b: &[u8], at: &mut usize) -> Option<usize> {
|
||||
let mut byte = || {
|
||||
*at += 1;
|
||||
b.get(*at - 1).copied().map(usize::from)
|
||||
};
|
||||
Some(match byte()? {
|
||||
253 => byte()? << 8 | byte()?,
|
||||
254 => byte()? + 253 * 2,
|
||||
255 => byte()? + 253,
|
||||
v => v,
|
||||
})
|
||||
Some(glyf)
|
||||
}
|
||||
|
||||
/// WOFF2's UIntBase128.
|
||||
@@ -368,7 +278,19 @@ fn glyf_points_fit(t: &[u8]) -> Option<bool> {
|
||||
let n = u16::from_be_bytes(contours.get(2 * g..2 * g + 2)?.try_into().ok()?);
|
||||
let mut total = 0usize;
|
||||
for _ in 0..if n == 0xFFFF { 0 } else { n } {
|
||||
total += read_255_u16(points, &mut at)?;
|
||||
// 255UInt16
|
||||
let code = *points.get(at)?;
|
||||
at += 1;
|
||||
let mut byte = || {
|
||||
at += 1;
|
||||
points.get(at - 1).copied().map(usize::from)
|
||||
};
|
||||
total += match code {
|
||||
253 => byte()? << 8 | byte()?,
|
||||
254 => byte()? + 253 * 2,
|
||||
255 => byte()? + 253,
|
||||
v => usize::from(v),
|
||||
};
|
||||
if total > 1 << 16 {
|
||||
return Some(false);
|
||||
}
|
||||
|
||||
@@ -455,50 +455,3 @@ fn fonts_stored_at_once_under_one_name_each_get_their_own_file() {
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A WOFF2 collection of `tables` untransformed tables whose fonts list the tables in `fonts`.
|
||||
fn woff2_collection(tables: usize, fonts: &[Vec<u16>]) -> Vec<u8> {
|
||||
fn u255(v: u16, out: &mut Vec<u8>) {
|
||||
match v {
|
||||
0..=252 => out.push(v as u8),
|
||||
_ => {
|
||||
out.push(253);
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut dir = Vec::new();
|
||||
for i in 0..tables {
|
||||
dir.push(63);
|
||||
dir.extend_from_slice(&(i as u32).to_be_bytes()); // a made-up tag per table
|
||||
base128(4, &mut dir);
|
||||
}
|
||||
dir.extend_from_slice(&0x0001_0000u32.to_be_bytes());
|
||||
u255(fonts.len() as u16, &mut dir);
|
||||
for font in fonts {
|
||||
u255(font.len() as u16, &mut dir);
|
||||
dir.extend_from_slice(&[0, 1, 0, 0]);
|
||||
for &i in font {
|
||||
u255(i, &mut dir);
|
||||
}
|
||||
}
|
||||
woff2_file(b"ttcf", tables, &dir, &vec![0; 4 * tables], 4 * tables + 1024)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_woff2_collection_rebuilding_huge_directories_is_refused_before_decoding() {
|
||||
// Each font's rebuilt directory takes 16 bytes per table it lists, the file about two: 600
|
||||
// fonts listing 2,000 tables each come to 19 MB of directories from a 2.4 MB file.
|
||||
let all: Vec<u16> = (0..2_000).collect();
|
||||
let file = woff2_collection(2_000, &vec![all; 600]);
|
||||
assert_eq!(decode_font_file(&file), Err(UserFontError::BadWebFont(BAD_COLLECTION.into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_woff2_collection_font_listing_a_table_twice_is_refused() {
|
||||
let file = woff2_collection(3, &[vec![0, 1, 2], vec![0, 1, 1]]);
|
||||
assert_eq!(decode_font_file(&file), Err(UserFontError::BadWebFont(BAD_COLLECTION.into())));
|
||||
// Fonts sharing tables are what collections are for: passed on to the decoder.
|
||||
let file = woff2_collection(3, &[vec![0, 1, 2], vec![0, 1, 2]]);
|
||||
assert_ne!(decode_font_file(&file), Err(UserFontError::BadWebFont(BAD_COLLECTION.into())));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user