aleleba/wordcraft
1
0
Fork 0
mirror of https://github.com/storytold/wordcraft.git synced 2026-10-27 20:07:19 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3873 from benandez/sec/h

Browse Source 
Security: import/export format fixes — LaTeX verbatim escape, Markdown link escaping, importer limits (#3713, #3714, #3705)
This commit is contained in:
Dylan M. Taylor
2026-10-25 21:09:20 -04:00
committed by GitHub
parent 6689d186f6 22df7caf4d
commit 0ecfbb57f9
11 changed files with 662 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/formats/src/html.rs
+19 -3
View File
@@ -724,7 +724,7 @@ fn apply_style(css: &str, f: &mut Fmt, align: &mut Option<Align>, preserve: &mut
if model::is_mono(&fam) || fam.eq_ignore_ascii_case("monospace") {
f.code = true;
} else if !fam.is_empty() && !matches!(fam.to_ascii_lowercase().as_str(), "serif" | "sans-serif" | "inherit" | "initial") {
f.font = Some(fam);
f.font = model::kept_font(&fam).or(f.font.take());
}
}
"vertical-align" => {
@@ -1226,7 +1226,7 @@ impl<'r> Builder<'r> {
if model::is_mono(&fam) {
fmt.code = true;
} else if !fam.is_empty() {
fmt.font = Some(fam);
fmt.font = model::kept_font(&fam).or(fmt.font.take());
}
}
if let Some(sz) = attr(attrs, "size") {
@@ -1245,7 +1245,7 @@ impl<'r> Builder<'r> {
}
"a" => {
if let Some(h) = attr(attrs, "href").filter(|h| wordcraft_doc::link::is_safe_href(h)) {
fmt.link = Some(h.trim().to_string());
fmt.link = model::kept_link(h.trim());
}
}
_ => {}
@@ -1588,17 +1588,20 @@ fn collapse_ws(s: &str) -> String {
/// Parse HTML into a document.
pub fn import(s: &str) -> Document {
model::begin_import();
model::to_doc(&parse(s))
}
/// [`import`], loading pictures that aren't `data:` URIs through `images`.
pub fn import_with(s: &str, images: ImageLoader) -> Document {
model::begin_import();
model::to_doc(&parse_with(s, images))
}
/// [`import_with`] onto `page` (its size and margins), which tables size to: HTML gives no page
/// size, so the caller picks it (the region's paper, #1381).
pub fn import_on(s: &str, images: ImageLoader, page: &wordcraft_doc::SectionProps) -> Document {
model::begin_import();
model::to_doc_in(&parse_with(s, images), page.clone())
}
@@ -2424,4 +2427,17 @@ mod tests {
assert!(html.contains("vertical-align:middle"));
assert!(html.contains("vertical-align:bottom"));
}
#[test]
fn many_formatted_leading_breaks_trim_quickly() {
// #3705: a paragraph opening with alternating bold and italic `<br>` was trimmed one
// span at a time with `Vec::remove(0)`.
let html = format!("<p>{}x</p>", "<b><br></b><i><br></i>".repeat(100_000));
let t = std::time::Instant::now();
let f = parse(&html);
// About a second on a loaded machine; trimming one span at a time ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed());
let texts: Vec<String> = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect();
assert_eq!(texts, ["x"]);
}
}
crates/formats/src/latex.rs
+51 -12
View File
@@ -243,8 +243,10 @@ impl Writer {
}
self.out.push_str("\\begin{verbatim}\n");
for l in lines {
// The environment ends at the first `\end{verbatim}`, so keep it from appearing.
let l: String = l.replace("\\end{verbatim}", "\\end {verbatim}").chars().filter(|c| *c == '\t' || !c.is_control()).collect();
// The environment ends at the first `\end{verbatim}`, so keep it from appearing. The
// control characters go first: removing one afterwards could join a delimiter back up.
let l: String = l.chars().filter(|c| *c == '\t' || !c.is_control()).collect();
let l = l.replace("\\end{verbatim}", "\\end {verbatim}");
self.out.push_str(&l);
self.out.push('\n');
}
@@ -1055,6 +1057,7 @@ fn math_seq(t: &[MTok], pos: &mut usize, depth: usize, in_opt: bool) -> String {
/// Import a LaTeX source file.
pub fn import(bytes: &[u8]) -> Document {
model::begin_import();
let mut d = model::to_doc(&parse(&crate::txt::decode(bytes.get(..MAX_SOURCE).unwrap_or(bytes))));
d.ensure_nonempty();
d
@@ -2333,20 +2336,21 @@ impl<'a> Parser<'a> {
}
}
"href" => {
let url = self.arg_raw();
self.with_arg(move |f| f.link = Some(url));
let url = model::kept_link(&self.arg_raw());
self.with_arg(move |f| f.link = url);
}
"url" | "nolinkurl" => {
let url = self.arg_raw();
let mut f = self.fmt();
if name == "url" {
f.link = Some(url.clone());
f.link = model::kept_link(&url);
}
self.para.push_text(&url, &f);
}
"hyperlink" => {
let target = self.arg_raw();
self.with_arg(move |f| f.link = Some(format!("#{target}")));
let link = model::kept_link(&format!("#{target}"));
self.with_arg(move |f| f.link = link);
}
"hypertarget" => {
let target = self.arg_raw();
@@ -2360,7 +2364,7 @@ impl<'a> Parser<'a> {
"ref" | "eqref" | "pageref" | "autoref" | "cref" | "Cref" | "nameref" => {
let l = self.arg_raw();
let mut f = self.fmt();
f.link = Some(format!("#{l}"));
f.link = model::kept_link(&format!("#{l}"));
self.para.push_text(&l, &f);
}
"cite" | "citep" | "citet" | "parencite" | "textcite" | "autocite" => {
@@ -2671,19 +2675,24 @@ fn table(toks: &[Tok], spec: &str, depth: usize) -> FTable {
for cell_toks in r {
let mut cell = Cell::default();
let mut content = trim_cell(cell_toks);
// `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`.
// `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`. Each wrapper
// rescans its content, so only MAX_NEST of them are unwrapped (a cell nests one or
// two); further ones stay in the content.
let mut sub = Parser::new(content, depth + 1);
let mut wrappers = 0usize;
loop {
sub.skip_spaces();
match sub.t.get(sub.pos) {
Some(Tok::Cmd(c)) if c == "multicolumn" => {
Some(Tok::Cmd(c)) if c == "multicolumn" && wrappers < MAX_NEST => {
wrappers += 1;
sub.pos += 1;
cell.colspan = sub.arg_raw().trim().parse::<u32>().unwrap_or(1).clamp(1, 63);
let _ = sub.arg_raw();
content = sub.arg_tokens();
sub = Parser::new(content, depth + 1);
}
Some(Tok::Cmd(c)) if c == "multirow" || c == "multirow*" => {
Some(Tok::Cmd(c)) if (c == "multirow" || c == "multirow*") && wrappers < MAX_NEST => {
wrappers += 1;
sub.pos += 1;
let _ = sub.opt_arg();
cell.rowspan = sub.arg_raw().trim().parse::<i64>().map(|n| n.unsigned_abs().clamp(1, 1000) as u32).unwrap_or(1);
@@ -3161,13 +3170,13 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after.
let FBlock::Para(p) = &flow.blocks[0] else { panic!() };
assert_eq!(
p.inlines[0],
Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() })
Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() })
);
assert_eq!(
p.inlines[2],
Inline::Text(
"https://example.com/~user/page".to_string(),
Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() }
Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() }
)
);
}
@@ -3234,4 +3243,34 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after.
let Some(Block::Table(bt_bordered)) = back_bordered.body.first().map(|b| &**b) else { panic!() };
assert!(bt_bordered.props.style.is_some(), "imported table should remain bordered");
}
#[test]
fn verbatim_end_is_kept_out_after_control_characters_go() {
// A control character inside `\end{verbatim}` was removed after the check for it, so
// the code line closed the environment and the rest ran as TeX.
let mut d = Document::new();
let line = "\\end{verb\u{E}atim}\\input{/etc/passwd}";
d.body = vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(line, Default::default()).styled(model::CODE_STYLE))];
let out = export(&d);
assert_eq!(out.matches("\\end{verbatim}").count(), 1, "{out}");
assert!(out.contains("\\end {verbatim}\\input"), "{out}");
}
#[test]
fn nested_multicolumn_wrappers_are_bounded() {
// Each `\multicolumn` unwrapped rescanned the rest of the cell; nested ones were all
// unwrapped, quadratic in the nesting.
let n = 50_000;
let src = format!("\\begin{{tabular}}{{l}}{}x{}\\\\\\end{{tabular}}", "\\multicolumn{1}{c}{".repeat(n), "}".repeat(n));
let t = std::time::Instant::now();
let f = parse(&src);
// Under a second on a loaded machine; unwrapping every wrapper took 213 s.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert!(f.blocks.iter().any(|b| matches!(b, FBlock::Table(_))));
// One or two wrappers, as written in real tables, are still read.
let f = parse("\\begin{tabular}{ll}\\multicolumn{2}{c}{\\multirow{2}{*}{Both}}\\\\\\end{tabular}");
let Some(FBlock::Table(t)) = f.blocks.first() else { panic!("{:?}", f.blocks) };
assert_eq!((t.rows[0][0].colspan, t.rows[0][0].rowspan), (2, 2));
assert_eq!(t.rows[0][0].text(), "Both");
}
}
crates/formats/src/lib.rs
+2
View File
@@ -91,5 +91,7 @@ pub fn export_with(ext: &str, doc: &Document, opts: &ExportOptions) -> Option<Re
Some(out.map(|bytes| comments::append(&ext, doc, bytes)))
}
#[cfg(test)]
mod test_log;
#[cfg(test)]
mod tests;
crates/formats/src/markdown.rs
+117 -27
View File
@@ -10,6 +10,7 @@
//! Export writes the same constructs; pictures become `![alt](media-name)` placeholders.
use std::collections::HashMap;
use std::sync::Arc;
use wordcraft_doc::{Align, Document};
@@ -24,16 +25,19 @@ const MAX_NEST: usize = 32;
/// Parse Markdown text into a document.
pub fn import(text: &str) -> Document {
model::begin_import();
model::to_doc(&parse(text))
}
/// [`import`] onto `page`: Markdown gives no page size (#1381).
pub fn import_on(text: &str, page: &wordcraft_doc::SectionProps) -> Document {
model::begin_import();
model::to_doc_in(&parse(text), page.clone())
}
/// Link reference definitions: the normalised label to its destination.
pub type Refs = HashMap<String, String>;
/// A destination is shared by every reference to it, not copied into each (#3713).
pub type Refs = HashMap<String, Arc<str>>;
/// A label as CommonMark compares them: case-folded, and any run of white space one space
/// (§4.7, "matches"). `[FOO bar]` and `[foo BAR]` are the same label.
@@ -141,7 +145,7 @@ fn take_definitions(lines: &mut Vec<String>) -> Refs {
if starts_block && let Some((label, dest)) = link_definition(line) {
// The first definition of a label wins, as CommonMark says, and definitions may
// follow one another.
refs.entry(label).or_insert(dest);
refs.entry(label).or_insert(dest.into());
return false;
}
starts_block = is_blank(line) || atx_heading(t).is_some() || thematic_break(t);
@@ -799,10 +803,18 @@ fn link_dest(s: &str, at: usize) -> Option<(String, usize)> {
i += ws;
let r = s.get(i..)?;
let (dest, used) = if let Some(inner) = r.strip_prefix('<') {
let end = inner.find(['>', '\n'])?;
if inner.get(end..)?.starts_with('\n') {
return None;
}
// The first `>` not escaped by a backslash ends it; a line break first means no link.
let b = inner.as_bytes();
let mut k = 0;
let end = loop {
match b.get(k)? {
b'\\' => k += 1,
b'>' => break k,
b'\n' => return None,
_ => {}
}
k += 1;
};
(inner.get(..end)?.to_string(), end + 2)
} else {
let mut depth = 0i32;
@@ -983,7 +995,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
let dest = link_dest(s, close + 1);
scan_budget = scan_budget.saturating_sub(dest.as_ref().map_or(s.len(), |d| d.1).saturating_sub(close));
let (dest, end) = dest?;
return Some((text.to_string(), dest, end));
return Some((text.to_string(), Arc::from(dest), end));
}
// A reference link: `[text][label]`, `[text][]` and `[text]` all take
// their destination from a definition collected before parsing (#3238).
@@ -1020,7 +1032,11 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
Some((inner, dest, end)) if base.link.is_none() => {
flush!();
let mut f = base.clone();
f.link = Some(dest);
// A destination too long to keep leaves its text unlinked.
f.link = (dest.len() <= model::MAX_LINK).then_some(dest);
if f.link.is_none() {
model::dropped(model::Dropped::Link);
}
for x in parse_inlines(&inner, &f, depth + 1, refs) {
nodes.push(Node::Inl(x));
}
@@ -1047,7 +1063,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
if is_auto {
flush!();
let mut f = base.clone();
f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}") } else { inner.to_string() });
f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}").into() } else { inner.into() });
push_text(&mut nodes, inner, &f);
i += inner.len() + 2;
text_start = i;
@@ -1202,18 +1218,16 @@ fn emit(mut nodes: Vec<Node>) -> Vec<Inline> {
}
out.push(Inline::Text(t.to_string(), f));
};
let base_of = |nodes: &[Node], k: usize| -> Fmt {
// Delimiter literals take the formatting (link, …) of neighbouring text.
for j in (0..k).rev() {
if let Some(Node::Text(_, f)) = nodes.get(j) {
return Fmt { bold: false, italic: false, strike: false, ..f.clone() };
}
}
Fmt::default()
};
for (k, n) in nodes.iter().enumerate() {
// Delimiter literals take the formatting (link, …) of the text before them: the last text
// node, tracked here rather than searched for backwards from every delimiter (quadratic on
// a paragraph of delimiters).
let mut last_text: Option<&Fmt> = None;
for n in &nodes {
match n {
Node::Text(t, f) => push(&mut out, t, f, em, strong, strike),
Node::Text(t, f) => {
push(&mut out, t, f, em, strong, strike);
last_text = Some(f);
}
Node::Inl(Inline::Text(t, f)) => push(&mut out, t, f, em, strong, strike),
Node::Inl(other) => out.push(other.clone()),
Node::Delim { ch, left, opens, closes, .. } => {
@@ -1224,9 +1238,11 @@ fn emit(mut nodes: Vec<Node>) -> Vec<Inline> {
Emph::Strike => strike = strike.saturating_sub(1),
}
}
let lit: String = std::iter::repeat_n(*ch, *left).collect();
let f = base_of(&nodes, k);
push(&mut out, &lit, &f, em, strong, strike);
if *left > 0 {
let lit: String = std::iter::repeat_n(*ch, *left).collect();
let f = last_text.map_or_else(Fmt::default, |f| Fmt { bold: false, italic: false, strike: false, ..f.clone() });
push(&mut out, &lit, &f, em, strong, strike);
}
for e in opens {
match e {
Emph::Em => em += 1,
@@ -1378,7 +1394,7 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String {
}
let mut out = String::new();
let mut stack: Vec<Mark> = Vec::new();
let mut link: Option<String> = None;
let mut link: Option<Arc<str>> = None;
let close_all = |out: &mut String, stack: &mut Vec<Mark>| {
while let Some(m) = stack.pop() {
out.push_str(marker(m));
@@ -1436,8 +1452,35 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String {
out
}
/// `l` as a Markdown link destination that reads back as exactly `l`, the target
/// `is_safe_href` passed. What Markdown would decode (a backslash escape, an entity reference)
/// or end the destination on (`<`, `>`) is backslash-escaped, never deleted: deleting `<` or `>`
/// could turn an accepted relative target into `javascript:…` or `//host`. Control characters
/// can't be written in a destination and are left out, as the safety check ignores them.
fn link_target(l: &str) -> String {
if l.contains([' ', '(', ')', '<', '>']) { format!("<{}>", l.replace(['<', '>', '\n'], "")) } else { l.to_string() }
let l: String = l.chars().filter(|c| !c.is_ascii_control()).collect();
let mut out = String::with_capacity(l.len() + 2);
for (i, c) in l.char_indices() {
let rest = l.get(i + c.len_utf8()..).unwrap_or("");
match c {
'<' | '>' => out.push('\\'),
// A backslash is literal unless punctuation (or the closing `)`/`>`) follows it.
'\\' if rest.chars().next().is_none_or(|n| n.is_ascii_punctuation()) => out.push('\\'),
'&' if entity_like(rest) => out.push('\\'),
_ => {}
}
out.push(c);
}
if l.contains(|c: char| c.is_whitespace() || matches!(c, '(' | ')' | '<' | '>')) { format!("<{out}>") } else { out }
}
/// Whether `rest` (the text after a `&`) starts like an entity reference (`amp;`, `#106;`,
/// `#x6A;`), which a Markdown reader would decode.
fn entity_like(rest: &str) -> bool {
let Some(end) = within(rest, 34).find(';') else { return false };
let body = rest.get(..end).unwrap_or("");
let body = body.strip_prefix('#').map_or(body, |n| n.strip_prefix(['x', 'X']).unwrap_or(n));
!body.is_empty() && body.chars().all(|c| c.is_ascii_alphanumeric())
}
fn marker(m: Mark) -> &'static str {
@@ -1640,7 +1683,7 @@ mod tests {
let s = inl("see [the *site*](http://x.org \"T\") and `a*b*` \\*lit\\* &amp; <https://y.com>");
let link: Vec<_> = s
.iter()
.filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.clone(), f.italic)) } else { None })
.filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.to_string(), f.italic)) } else { None })
.collect();
assert_eq!(link[0], ("the ".into(), "http://x.org".into(), false));
assert_eq!(link[1], ("site".into(), "http://x.org".into(), true));
@@ -1661,7 +1704,7 @@ mod tests {
.inlines
.iter()
.filter_map(|x| match x {
Inline::Text(t, f) => f.link.clone().map(|l| (t.clone(), l)),
Inline::Text(t, f) => f.link.as_deref().map(|l| (t.clone(), l.to_string())),
_ => None,
})
.collect(),
@@ -2152,4 +2195,51 @@ mod tests {
let texts: Vec<String> = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect();
assert_eq!(texts, ["Hello world!", "After comment."]);
}
#[test]
fn a_paragraph_of_delimiters_parses_in_linear_time() {
// #3714: each delimiter's literal looked back for the text before it, to the start of
// the paragraph when there was none (quadratic).
let src = "*_".repeat(200_000);
let t = std::time::Instant::now();
let out = inl(&src);
// Under a second on a loaded machine; the backwards search ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
let text: String = out.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect();
assert!(text.chars().all(|c| matches!(c, '*' | '_')), "only delimiters come out");
// Literal delimiters still take the link of the text before them.
let s = inl("[a](http://x.org) [b*](http://y.org)");
assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "b*" && f.link.as_deref() == Some("http://y.org"))), "{s:?}");
}
#[test]
fn exported_link_destinations_read_back_unchanged() {
// Removing `<` and `>` from a destination after the safety check could make a relative
// target a script or network-path one (`java<script:…` → `javascript:…`). They are
// escaped instead, as are backslash escapes and entity references a reader would decode.
let md = |l: &str| inlines_md(&[Inline::Text("x".into(), Fmt { link: Some(l.into()), ..Default::default() })]);
assert_eq!(md("java<script:alert(1)"), "[x](<java\\<script:alert(1)>)");
for l in [
"java<script:alert(1)",
"/</evil.example/(x)",
"a>b c",
"&#106;avascript:alert(1)",
"&#x6A;avascript:x",
"&amp;",
"javascript\\:x",
"dir\\",
"dir\\(1)\\",
"q?a=1&b=2",
] {
let out = md(l);
let back = inl(&out);
let got: Vec<_> =
back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect();
assert_eq!(got, [l.to_string()], "{l:?} → {out:?}");
}
// Ordinary destinations are written as before.
assert_eq!(md("https://example.com/a?b=1&c=2"), "[x](https://example.com/a?b=1&c=2)");
assert_eq!(md("https://example.com/a b"), "[x](<https://example.com/a b>)");
assert_eq!(md("https://e.com/x\ny"), "[x](https://e.com/xy)");
}
}
crates/formats/src/model.rs
+140 -18
View File
@@ -19,6 +19,84 @@ pub const MONO_FONT: &str = "Courier New";
pub const CODE_STYLE: &str = "Code";
/// Nesting limit for tables in tables (hostile input).
pub const MAX_DEPTH: usize = 8;
/// Longest font name the importers keep, in bytes (as the .docx reader); a longer one (no real
/// font has one) is dropped (#3713).
pub const MAX_FONT_NAME: usize = 256;
/// Longest hyperlink target the importers keep, in bytes; a longer link is dropped and its text
/// kept (#3713).
pub const MAX_LINK: usize = 8 << 10;
/// Link and font bytes the document's runs may carry between them. A flow's spans share one
/// copy of a name ([`Fmt`]), but every run of a document owns its own: past this budget, runs
/// lose their link and font rather than multiply a small file into gigabytes (#3713).
const MAX_RUN_STRINGS: usize = 64 << 20;
/// `name` as a span's font, unless it is longer than [`MAX_FONT_NAME`] (then reported, see
/// [`dropped`]).
pub fn kept_font(name: &str) -> Option<Arc<str>> {
let kept = (name.len() <= MAX_FONT_NAME).then(|| name.into());
if kept.is_none() {
dropped(Dropped::Font);
}
kept
}
/// `target` as a span's link, unless it is longer than [`MAX_LINK`] (then reported, see
/// [`dropped`]).
pub fn kept_link(target: &str) -> Option<Arc<str>> {
let kept = (target.len() <= MAX_LINK).then(|| target.into());
if kept.is_none() {
dropped(Dropped::Link);
}
kept
}
/// What an importer leaves out to stay within a limit (review of #3873: no limit drops data
/// without saying so).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Dropped {
/// A font name over [`MAX_FONT_NAME`].
Font,
/// A link target over [`MAX_LINK`].
Link,
/// Runs' links and fonts past [`MAX_RUN_STRINGS`].
RunStrings,
/// A style name cut to [`MAX_FONT_NAME`].
StyleName,
}
impl Dropped {
fn message(self) -> String {
match self {
Dropped::Font => format!("import: font names longer than {MAX_FONT_NAME} bytes are dropped; that text keeps the surrounding font"),
Dropped::Link => format!("import: links longer than {MAX_LINK} bytes are dropped; their text is kept"),
Dropped::RunStrings => {
format!("import: the document's runs carry more than {MAX_RUN_STRINGS} bytes of links and font names; later runs lose them")
}
Dropped::StyleName => format!("import: style names longer than {MAX_FONT_NAME} bytes are cut short"),
}
}
}
thread_local! {
/// The kinds of [`Dropped`] already reported for the document being imported on this thread.
/// Per thread because a document is parsed and built on one, and the parsers drop things
/// from places that have no log of their own to hand.
static REPORTED: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };
}
/// Log that the importer left something of `kind` out: once per kind per imported document
/// ([`begin_import`]), not once per item.
pub(crate) fn dropped(kind: Dropped) {
let bit = 1u8 << (kind as u8);
if REPORTED.with(|r| r.replace(r.get() | bit)) & bit == 0 {
log::warn!("{}", kind.message());
}
}
/// Start a new document's reports: each importer's entry point calls this first.
pub(crate) fn begin_import() {
REPORTED.with(|r| r.set(0));
}
/// Character formatting the text formats can express.
#[derive(Clone, Debug, Default, PartialEq)]
@@ -30,11 +108,13 @@ pub struct Fmt {
pub code: bool,
pub sup: bool,
pub sub: bool,
pub link: Option<String>,
/// Shared: every span of a link holds the same target.
pub link: Option<Arc<str>>,
pub color: Option<Rgb>,
/// Points.
pub size: Option<f32>,
pub font: Option<String>,
/// Shared, as `link`.
pub font: Option<Arc<str>>,
/// Background (highlight / shading).
pub background: Option<Rgb>,
/// Tracked insertion of this text (formats that keep revisions: RTF).
@@ -239,15 +319,13 @@ impl Para {
/// Remove the characters in `start` from the start of the paragraph's text and those in
/// `end` from its end.
pub fn trim_with(&mut self, start: &[char], end: &[char]) {
while let Some(Inline::Text(t, _)) = self.inlines.first_mut() {
let n = t.trim_start_matches(start).len();
let cut = t.len() - n;
// The leading spans that trim to nothing go in one `drain`: removing them one at a time
// shifted the rest each time, quadratic in a run of separately formatted line breaks.
let k = self.inlines.iter().take_while(|i| matches!(i, Inline::Text(t, _) if t.trim_start_matches(start).is_empty())).count();
self.inlines.drain(..k);
if let Some(Inline::Text(t, _)) = self.inlines.first_mut() {
let cut = t.len() - t.trim_start_matches(start).len();
t.replace_range(..cut, "");
if t.is_empty() {
self.inlines.remove(0);
} else {
break;
}
}
while let Some(Inline::Text(t, _)) = self.inlines.last_mut() {
let n = t.trim_end_matches(end).len();
@@ -431,10 +509,10 @@ pub fn char_props(f: &Fmt) -> CharProps {
} else {
None
},
link: f.link.clone().filter(|l| !l.is_empty()),
link: f.link.as_deref().filter(|l| !l.is_empty()).map(str::to_string),
color: f.color.map(TextColor::Rgb),
size: f.size.filter(|s| s.is_finite() && *s >= 1.0 && *s <= 1638.0),
font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.clone().filter(|s| !s.is_empty()) },
font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.as_deref().filter(|s| !s.is_empty()).map(str::to_string) },
shading: f.background,
..Default::default()
}
@@ -518,6 +596,8 @@ struct Builder<'a> {
doc: &'a mut Document,
/// Revisions already in `doc` by (kind, author, date), so each change is added once.
revs: std::collections::HashMap<(u8, RevMark), u32>,
/// Link and font bytes given to runs so far ([`MAX_RUN_STRINGS`]).
run_strings: usize,
}
fn rev_key(kind: RevisionKind) -> u8 {
@@ -537,7 +617,7 @@ impl<'a> Builder<'a> {
revs.entry((rev_key(r.kind), mark)).or_insert(i as u32);
}
}
Builder { doc, revs }
Builder { doc, revs, run_strings: 0 }
}
/// The document revision for a tracked change, added the first time it is seen.
@@ -552,6 +632,22 @@ impl<'a> Builder<'a> {
i
}
/// `cp` as a run may keep it: without its link and font once the runs' copies of them are
/// over budget.
fn budget(&mut self, mut cp: CharProps) -> CharProps {
let n = cp.link.as_ref().map_or(0, String::len).saturating_add(cp.font.as_ref().map_or(0, String::len));
match self.run_strings.checked_add(n).filter(|&t| t <= MAX_RUN_STRINGS) {
Some(t) => self.run_strings = t,
None => {
if cp.link.is_some() || cp.font.is_some() {
dropped(Dropped::RunStrings);
}
(cp.link, cp.font) = (None, None);
}
}
cp
}
fn para(&mut self, p: &Para, list_num: Option<u32>) -> Paragraph {
let mut out = Paragraph::new();
out.props = para_props(p, list_num);
@@ -577,11 +673,12 @@ impl<'a> Builder<'a> {
match i {
Inline::Text(t, f) => {
let t = clean_text(t);
let mut cp = char_props(f);
let mut cp = self.budget(char_props(f));
cp.ins = f.ins.as_ref().map(|m| self.revision(RevisionKind::Insert, m));
cp.del = f.del.as_ref().map(|m| self.revision(RevisionKind::Delete, m));
if let Some(c) = &f.fmt_change {
cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), char_props(&c.old));
let old = self.budget(char_props(&c.old));
cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), old);
}
if p.kind == Kind::Code {
// Code's font comes from its style, before a change as after it.
@@ -616,7 +713,7 @@ impl<'a> Builder<'a> {
}
Inline::Ruby { base, ruby, fmt } => {
let (base, ruby) = (clean_text(base), clean_text(ruby));
let cp = char_props(fmt);
let cp = self.budget(char_props(fmt));
// A source without a base or without a reading (malformed markup) is just its
// text: a guide that annotates nothing is no guide.
if base.is_empty() || ruby.is_empty() {
@@ -883,13 +980,13 @@ fn fmt_of(doc: &Document, cp: &CharProps, in_code: bool) -> Fmt {
code,
sup: e.vert_align == Some(VertAlign::Superscript),
sub: e.vert_align == Some(VertAlign::Subscript),
link: e.link.clone(),
link: e.link.as_deref().map(Arc::from),
color: match e.color {
Some(TextColor::Rgb(c)) => Some(c),
_ => None,
},
size: e.size,
font: if code || in_code { None } else { e.shown_font().map(str::to_string) },
font: if code || in_code { None } else { e.shown_font().map(Arc::from) },
background: e.highlight.and_then(|h| h.rgb()).or(e.shading),
ins: None,
del: None,
@@ -1558,4 +1655,29 @@ mod tests {
assert_eq!(t.rows[1].len(), 2);
assert!(t.rows[1][0].covered);
}
#[test]
fn trim_drops_a_long_run_of_blank_spans_in_linear_time() {
// Separately formatted line breaks don't merge; each used to be removed with
// `Vec::remove(0)`, shifting all the rest (quadratic).
let mut p = Para::default();
let (b, i) = (Fmt { bold: true, ..Default::default() }, Fmt { italic: true, ..Default::default() });
for _ in 0..200_000 {
p.push_text("\n", &b);
p.push_text(" ", &i);
}
p.push_text(" x ", &Fmt::default());
let t = std::time::Instant::now();
p.trim();
// A fraction of a second; `Vec::remove(0)` per span ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert_eq!(p.inlines, vec![Inline::Text("x".into(), Fmt::default())]);
// The first span left keeps its formatting and loses only its leading blanks.
let mut p = Para::default();
p.push_text(" \t", &b);
p.push_text(" a ", &i);
p.push_text("b ", &b);
p.trim();
assert_eq!(p.inlines, vec![Inline::Text("a ".into(), i), Inline::Text("b".into(), b)]);
}
}
crates/formats/src/odt.rs
+6 -4
View File
@@ -19,6 +19,7 @@
use std::collections::HashMap;
use std::io::{Cursor, Read, Write};
use std::sync::Arc;
use quick_xml::XmlVersion;
use quick_xml::events::{BytesStart, Event};
@@ -1034,7 +1035,7 @@ struct FmtDelta {
color: Option<Rgb>,
background: Option<Rgb>,
size: Option<f32>,
font: Option<String>,
font: Option<Arc<str>>,
code: Option<bool>,
}
@@ -1340,7 +1341,7 @@ impl Styles {
if model::is_mono(&fam) || get(&a, "style:font-pitch").is_some_and(|p| p.trim().eq_ignore_ascii_case("fixed")) {
d.code = Some(true);
} else if !fam.is_empty() {
d.font = Some(fam);
d.font = model::kept_font(&fam);
}
}
}
@@ -1446,7 +1447,7 @@ impl Styles {
/// The document's default font: the Standard paragraph style's, else the default style's.
fn doc_font(&self) -> Option<String> {
self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.clone()).or_else(|| self.default_font.clone())
self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.as_deref().map(str::to_string)).or_else(|| self.default_font.clone())
}
fn chain<'a>(&'a self, name: &'a str) -> Vec<(&'a str, &'a StyleInfo)> {
@@ -2053,7 +2054,7 @@ impl Body<'_> {
"a" => {
let mut f = self.cur_fmt();
if let Some(h) = get(a, "xlink:href").filter(|h| !h.is_empty()) {
f.link = Some(h.to_string());
f.link = model::kept_link(h);
}
if let Some(s) = get(a, "text:style-name") {
f = self.styles.text_fmt(s, &f);
@@ -2626,6 +2627,7 @@ fn parse_tracked(bytes: &[u8]) -> Result<(Flow, SectionProps, Option<bool>, Opti
/// Parse an `.odt` package into a document.
pub fn import(bytes: &[u8]) -> Result<Document, String> {
model::begin_import();
let (flow, last, track, font, parts) = parse_tracked(bytes)?;
let mut doc = model::to_doc(&flow);
if font.is_some() {
crates/formats/src/rtf.rs
+17 -8
View File
@@ -19,6 +19,7 @@
//! tables. Group nesting and picture sizes are capped.
use std::collections::HashMap;
use std::sync::Arc;
use wordcraft_doc::props::VAlign;
use wordcraft_doc::{Align, Document, Rgb};
@@ -306,11 +307,13 @@ impl Writer {
"\\nosupersub"
});
}
let font = |f: &Fmt| if f.code { Some(model::MONO_FONT.to_string()) } else { f.font.clone() };
fn font(f: &Fmt) -> Option<&str> {
if f.code { Some(model::MONO_FONT) } else { f.font.as_deref() }
}
let old_font = font(old);
if old_font != font(cur) {
// No font is the document's default font, font 0 (Normal's).
let i = old_font.map_or(0, |n| self.t.font(&n));
let i = old_font.map_or(0, |n| self.t.font(n));
ctl.push_str(&format!("\\f{i}"));
}
let half_points = |s: Option<f32>| s.map(|s| (s * 2.0).round().clamp(2.0, 3276.0) as i32);
@@ -1106,7 +1109,7 @@ struct Reader {
st: St,
stack: Vec<St>,
overflow: usize,
fonts: HashMap<i32, String>,
fonts: HashMap<i32, Arc<str>>,
font_cur: i32,
font_name: String,
/// Code pages of the fonts whose charset names one (`\fcharset204` → 1251).
@@ -1398,7 +1401,9 @@ impl Reader {
fn commit_font(&mut self) {
let name = std::mem::take(&mut self.font_name).trim().trim_end_matches(';').trim().to_string();
if !name.is_empty() && self.fonts.len() < 4096 {
if let Some(name) = model::kept_font(&name).filter(|n| !n.is_empty())
&& self.fonts.len() < 4096
{
self.fonts.insert(self.font_cur, name);
if let Some(cp) = self.font_cp_cur.take() {
self.font_cps.insert(self.font_cur, cp);
@@ -1626,7 +1631,8 @@ impl Reader {
for c in s.chars() {
if c == ';' {
self.commit_font();
} else {
} else if self.font_name.len() <= model::MAX_FONT_NAME {
// One past the limit is enough for `commit_font` to drop the name.
self.font_name.push(c);
}
}
@@ -1646,8 +1652,10 @@ impl Reader {
for c in s.chars() {
if c == ';' {
self.commit_style();
} else {
} else if self.style_name.len() < model::MAX_FONT_NAME {
self.style_name.push(c);
} else {
model::dropped(model::Dropped::StyleName);
}
}
}
@@ -2329,7 +2337,7 @@ impl Reader {
"fldrslt" => {
let link = self.st.field.and_then(|i| self.fields.get(i)).and_then(|s| hyperlink(s));
if link.is_some() {
self.st.fmt.link = link;
self.st.fmt.link = link.as_deref().and_then(model::kept_link);
}
}
_ => {}
@@ -2807,7 +2815,7 @@ fn parse_full(bytes: &[u8]) -> Result<Parsed, String> {
if blocks.is_empty() {
blocks.push(FBlock::Para(Para::default()));
}
let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).cloned();
let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).map(|n| n.to_string());
Ok(Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers })
}
@@ -2850,6 +2858,7 @@ fn column_grid(rows: &mut [Vec<Cell>], rights: &[Vec<i32>]) -> Vec<f32> {
/// Parse RTF bytes into a document.
pub fn import(bytes: &[u8]) -> Result<Document, String> {
model::begin_import();
let Parsed { flow, last_section, default_font, headers_footers, .. } = parse_full(bytes)?;
let mut doc = model::to_doc_in(&flow, last_section);
// Each section's own headers and footers; those it leaves out stay linked to the previous.
crates/formats/src/sheet.rs
+138 -18
View File
@@ -18,7 +18,7 @@
//! all), sheets, rows, columns, cells and string lengths are capped, repeat counts are clamped to
//! what is left, and nothing recurses, so XML depth costs nothing.
use std::collections::{BTreeMap, HashMap};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{Cursor, Read};
use quick_xml::XmlVersion;
@@ -138,18 +138,28 @@ pub fn read_rows(bytes: &[u8], pick: Pick<'_>) -> Result<(Vec<String>, Vec<Vec<S
/// repeats numbered (`Name`, `Name 2`), ignoring case as merge fields do.
pub fn field_names(row: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::with_capacity(row.len());
// The names given so far, ASCII-lowercased, and per base name the next number to try: a set
// lookup per candidate, and each base resumes where it stopped, instead of rescanning every
// name from `2` for every repeat (cubic on a row of equal headers).
let mut taken: HashSet<String> = HashSet::with_capacity(row.len());
let mut next: HashMap<String, usize> = HashMap::new();
for (i, h) in row.iter().enumerate() {
let base = match h.trim() {
"" => format!("Column {}", i + 1),
t => t.to_string(),
};
let taken = |n: &str, out: &[String]| out.iter().any(|o| o.eq_ignore_ascii_case(n));
let mut name = base.clone();
let mut k = 2usize;
while taken(&name, &out) {
name = format!("{base} {k}");
k = k.saturating_add(1);
if taken.contains(&name.to_ascii_lowercase()) {
let k = next.entry(base.to_ascii_lowercase()).or_insert(2);
loop {
name = format!("{base} {k}");
*k = k.saturating_add(1);
if !taken.contains(&name.to_ascii_lowercase()) {
break;
}
}
}
taken.insert(name.to_ascii_lowercase());
out.push(name);
}
out
@@ -333,6 +343,8 @@ struct Grid {
rows: Vec<Vec<String>>,
cells: usize,
text: TextBudget,
/// A row has been left out for its text (and logged, once per sheet).
refused: bool,
}
impl Grid {
@@ -340,6 +352,15 @@ impl Grid {
self.rows.len() >= MAX_ROWS || self.cells >= MAX_CELLS || self.text.0 >= MAX_TEXT_BYTES
}
/// Note that a row is left out because its text is over what is left of [`MAX_TEXT_BYTES`]:
/// logged once per sheet, not once per row.
fn refuse(&mut self) {
if !self.refused {
self.refused = true;
log::warn!("sheet: rows with more text than the {MAX_TEXT_BYTES} byte budget has left are left out of the recipient list");
}
}
/// Keep `row` (trailing blanks trimmed) `times` times, as far as the caps allow.
fn push(&mut self, mut row: Vec<String>, times: u64) {
while row.last().is_some_and(String::is_empty) {
@@ -350,14 +371,15 @@ impl Grid {
}
let mut left = times;
while left > 0 && !self.full() {
let room = MAX_CELLS.saturating_sub(self.cells);
let mut r = row.clone();
r.truncate(room);
// What fits is measured on `row` itself; only a copy that will be kept is made.
let keep = row.len().min(MAX_CELLS.saturating_sub(self.cells));
let r = row.get(..keep).unwrap_or_default();
if !self.text.take(r.iter().map(String::len).sum()) {
self.refuse();
return;
}
self.cells = self.cells.saturating_add(r.len());
self.rows.push(r);
self.rows.push(r.to_vec());
left -= 1;
}
}
@@ -854,7 +876,9 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
// one is the wanted sheet.
let mut tables = 0usize;
let mut collecting = false;
let mut row: Option<(Vec<String>, u64, u64)> = None; // cells, pending blank cells, repeat
// Cells, pending blank cells, repeat, and the cells' text bytes (`None` once the row is over
// the text budget and so refused).
let mut row: Option<(Vec<String>, u64, u64, Option<usize>)> = None;
let mut cell: Option<OCell> = None;
let (mut in_p, mut skip) = (0usize, 0usize);
walk(xml, |ev| match ev {
@@ -875,7 +899,7 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
match n {
b"table-row" => {
let repeat = attr(e, b"number-rows-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1);
row = Some((Vec::new(), 0, repeat));
row = Some((Vec::new(), 0, repeat, Some(0)));
}
b"table-cell" | b"covered-table-cell" if row.is_some() => {
let repeat = attr(e, b"number-columns-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1);
@@ -959,22 +983,33 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
if n == b"covered-table-cell" {
skip = skip.saturating_sub(1);
}
let (Some(c), Some((cells, pending, _))) = (cell.take(), row.as_mut()) else { return };
let (Some(c), Some((cells, pending, _, budget))) = (cell.take(), row.as_mut()) else { return };
let Some(bytes) = budget else { return };
let text = ods_value(&c);
if text.is_empty() {
*pending = pending.saturating_add(c.repeat);
return;
}
// Blank cells before this one, then the value as often as it repeats, as far
// as the columns allow.
let blanks = (*pending).min(MAX_COLS.saturating_sub(cells.len()) as u64) as usize;
// as the columns and the grid's cells left (all `Grid::push` keeps) allow.
let room = MAX_COLS.min(MAX_CELLS.saturating_sub(grid.cells));
let blanks = (*pending).min(room.saturating_sub(cells.len()) as u64) as usize;
let times = c.repeat.min(room.saturating_sub(cells.len().saturating_add(blanks)) as u64) as usize;
// The row's text has to fit what is left of the budget to be kept at all
// (`Grid::push`): one that can't is refused now, before the copies are made.
*bytes = bytes.saturating_add(text.len().saturating_mul(times));
if *bytes > MAX_TEXT_BYTES.saturating_sub(grid.text.0) {
cells.clear();
*budget = None;
grid.refuse();
return;
}
cells.resize(cells.len() + blanks, String::new());
*pending = 0;
let times = c.repeat.min(MAX_COLS.saturating_sub(cells.len()) as u64) as usize;
cells.extend(std::iter::repeat_n(text, times));
}
b"table-row" => {
if let Some((cells, _, repeat)) = row.take() {
if let Some((cells, _, repeat, Some(_))) = row.take() {
grid.push(cells, repeat);
}
}
@@ -997,7 +1032,12 @@ fn ods_value(c: &OCell) -> String {
("time", Some(v)) => iso_time(v).unwrap_or_else(|| c.text.clone()),
("boolean", Some(v)) => if v.eq_ignore_ascii_case("true") || v == "1" { "TRUE" } else { "FALSE" }.into(),
("percentage" | "currency", Some(v)) if c.text.trim().is_empty() => v.parse::<f64>().map(number).unwrap_or_else(|_| c.text.clone()),
("string", Some(v)) if c.text.trim().is_empty() => v.to_string(),
("string", Some(v)) if c.text.trim().is_empty() => {
// Attribute text obeys the same per-cell cap as `<text:p>` text.
let mut s = String::new();
push_capped(&mut s, v);
s
}
_ => c.text.clone(),
}
}
@@ -1204,4 +1244,84 @@ mod tests {
assert_eq!(rows.len(), 1);
assert_eq!(rows[0], vec!["TRUE", "World", "Name"]);
}
#[test]
fn repeated_headers_are_numbered_in_linear_time() {
// Every repeat used to restart at `Name 2` and compare each candidate with every name
// so far: cubic in the number of equal headers.
let row = vec!["Name".to_string(); 5_000];
let t = std::time::Instant::now();
let names = field_names(&row);
// Milliseconds; the old numbering ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert_eq!(names.len(), 5_000);
assert_eq!(names[0], "Name");
assert_eq!(names[4_999], "Name 5000");
// Names a header already has stay collision-free, ignoring case.
let row: Vec<String> = ["Name", "Name 2", "Name", "name", "", "Column 5", "NAME 2"].iter().map(|s| s.to_string()).collect();
assert_eq!(field_names(&row), ["Name", "Name 2", "Name 3", "name 4", "Column 5", "Column 5 2", "NAME 2 2"]);
}
fn ods(rows: &str) -> Vec<u8> {
let content = format!(
r#"<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"><office:body><office:spreadsheet><table:table table:name="Sheet1">{rows}</table:table></office:spreadsheet></office:body></office:document-content>"#
);
zip(&[("mimetype", "application/vnd.oasis.opendocument.spreadsheet"), ("content.xml", &content)])
}
#[test]
fn ods_string_attributes_are_capped_before_repeating() {
// An `office:string-value` is a cell's text: it obeys MAX_CELL_CHARS like `<text:p>`
// does, so repeating it across the columns can't build a huge row first.
let long = "v".repeat(MAX_CELL_CHARS * 4);
let book = ods(&format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{long}" table:number-columns-repeated="3"/></table:table-row>"#
));
let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].len(), 3);
assert!(rows[0].iter().all(|c| c.chars().count() == MAX_CELL_CHARS));
// A row whose repeated text alone is over the text budget is dropped as it is built
// (it was built in full, then cloned, then refused); the rows after it still count.
let wide = "€".repeat(MAX_CELL_CHARS);
let book = ods(&format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row><table:table-row><table:table-cell><text:p>next</text:p></table:table-cell></table:table-row>"#
));
let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert_eq!(rows, vec![vec!["next".to_string()]]);
// The row left out is said to be (review of #3873).
assert_eq!(warned.iter().filter(|w| w.contains("are left out of the recipient list")).count(), 1, "{warned:?}");
}
#[test]
fn rows_over_the_text_budget_are_logged_once_per_sheet() {
let wide = "€".repeat(MAX_CELL_CHARS);
let over = format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row>"#
);
let book = ods(&format!("{over}{over}{over}<table:table-row><table:table-cell><text:p>next</text:p></table:table-cell></table:table-row>"));
let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert_eq!(rows, vec![vec!["next".to_string()]]);
assert_eq!(warned.iter().filter(|w| w.contains("recipient list")).count(), 1, "one warning, not one per row: {warned:?}");
let book = ods("<table:table-row><table:table-cell><text:p>fine</text:p></table:table-cell></table:table-row>");
let (_, warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert!(warned.is_empty(), "{warned:?}");
}
#[test]
fn ods_repeated_text_is_budgeted_for_the_cells_the_grid_keeps() {
// One cell left in the grid: of a long value repeated a thousand times only the first
// copy is kept, and it fits the text budget, so the row isn't refused for the others.
let wide = "€".repeat(MAX_CELL_CHARS);
let full = MAX_CELLS / MAX_COLS - 1;
let book = ods(&format!(
r#"<table:table-row table:number-rows-repeated="{full}"><table:table-cell office:value-type="string" office:string-value="x" table:number-columns-repeated="{MAX_COLS}"/></table:table-row><table:table-row><table:table-cell office:value-type="string" office:string-value="x" table:number-columns-repeated="{}"/></table:table-row><table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row>"#,
MAX_COLS - 1
));
let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap();
assert_eq!(rows.len(), full + 2);
assert_eq!(rows.last(), Some(&vec![wide]));
assert_eq!(rows.iter().map(Vec::len).sum::<usize>(), MAX_CELLS);
}
}
crates/formats/src/test_log.rs
+36
View File
@@ -0,0 +1,36 @@
//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that
//! a limit which leaves something out says so.
use std::cell::RefCell;
struct Capture;
thread_local! {
static LINES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
impl log::Log for Capture {
fn enabled(&self, m: &log::Metadata) -> bool {
m.level() <= log::Level::Warn
}
fn log(&self, r: &log::Record) {
if self.enabled(r.metadata()) {
LINES.with(|l| l.borrow_mut().push(r.args().to_string()));
}
}
fn flush(&self) {}
}
static CAPTURE: Capture = Capture;
/// `f`'s result and the warnings logged on this thread while it ran.
pub(crate) fn warnings<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
// Only this logger is ever installed in a test binary; a second call keeps it.
let _ = log::set_logger(&CAPTURE);
log::set_max_level(log::LevelFilter::Warn);
LINES.with(|l| l.borrow_mut().clear());
let out = f();
(out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut())))
}
crates/formats/src/tests.rs
+132 -5
View File
@@ -748,8 +748,11 @@ fn code_spans_and_links_around_backticks_read_as_before() {
// inside a code span in link text, and link text with a backtick that nothing closes.
let flow = crate::markdown::parse("\\``e` `` a ` b `` then ``` and `c` [x `]` y](http://a.org) [p ` q](http://b.org) ``f");
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
let spans: Vec<(String, bool, Option<String>)> =
p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.clone())) } else { None }).collect();
let spans: Vec<(String, bool, Option<String>)> = p
.inlines
.iter()
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.as_deref().map(str::to_string))) } else { None })
.collect();
let code: Vec<&str> = spans.iter().filter(|s| s.1).map(|s| s.0.as_str()).collect();
assert_eq!(code, ["e", "a ` b", "c", "]"], "{spans:?}");
let linked = |url: &str| spans.iter().filter(|s| s.2.as_deref() == Some(url)).map(|s| s.0.as_str()).collect::<String>();
@@ -766,7 +769,7 @@ fn latex_href_keeps_ampersand() {
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.clone()) } else { None })
.filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.as_deref().map(str::to_string)) } else { None })
.collect();
assert_eq!(links, [Some("https://e.com/a?b=1&c=2".to_string())]);
@@ -775,7 +778,7 @@ fn latex_href_keeps_ampersand() {
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.clone())) } else { None })
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.as_deref().map(str::to_string))) } else { None })
.collect();
let url = "https://e.com/a?b=1&c=2".to_string();
assert_eq!(texts, [(url.clone(), Some(url))]);
@@ -789,7 +792,8 @@ fn brackets_do_not_turn_later_links_into_text() {
let links = |md: &str| {
let flow = crate::markdown::parse(md);
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
let mut urls: Vec<String> = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.clone() } else { None }).collect();
let mut urls: Vec<String> =
p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect();
urls.dedup();
urls
};
@@ -863,6 +867,21 @@ fn exported_links_keep_only_safe_targets() {
}
}
/// A link target with any whitespace (a no-break or ideographic space too) is written in `<…>`,
/// so the reader, which ends a bare destination at the first whitespace, reads it back whole.
#[test]
fn markdown_link_targets_with_unicode_spaces_read_back() {
for l in ["https://e.com/a\u{a0}b", "https://e.com/a\u{3000}b", "https://e.com/a b"] {
let mut d = Document::new();
let mut p = Paragraph::with_text("", CharProps::default());
p.insert_text(0, "go", &CharProps { link: Some(l.to_string()), ..Default::default() }).unwrap();
d.body = vec![para_block(p)];
let md = String::from_utf8(export("md", &d).unwrap().unwrap()).unwrap();
let sp = spans(&crate::markdown::parse(&md));
assert_eq!(sp.iter().find(|(t, _)| t == "go").and_then(|(_, f)| f.link.as_deref()), Some(l), "md: {md}");
}
}
#[test]
fn hidden_drawings_are_left_out_of_exports() {
let mut d = wordcraft_doc::Document::from_text("x");
@@ -875,3 +894,111 @@ fn hidden_drawings_are_left_out_of_exports() {
// Two pictures, one hidden: one comes out.
assert_eq!(crate::html::export(&d).matches("<img").count(), 1);
}
/// Every text span of the flow, with its formatting.
fn spans(flow: &model::Flow) -> Vec<(String, model::Fmt)> {
flow.blocks
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.clone())) } else { None })
.collect()
}
/// A font name or link target too long for any real one is dropped, its text kept: every run
/// carried its own copy, so one long name times many runs was gigabytes (#3713).
#[test]
fn overlong_font_names_and_links_are_dropped() {
let long = "x".repeat(1 << 20);
let rtf = format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}}}\\f1 {}}}", "{\\b a}a".repeat(1000));
let flow = crate::rtf::parse(rtf.as_bytes()).unwrap();
let sp = spans(&flow);
assert!(sp.iter().map(|s| s.0.len()).sum::<usize>() >= 2000);
assert!(sp.iter().all(|(_, f)| f.font.is_none()), "an overlong RTF font name is no font");
// A real one still is.
let flow = crate::rtf::parse(b"{\\rtf1{\\fonttbl{\\f0 Arial;}{\\f1 Georgia;}}\\f1 a}").unwrap();
assert_eq!(spans(&flow)[0].1.font.as_deref(), Some("Georgia"));
let html =
format!("<p><span style=\"font-family:'{long}'\"><a href=\"https://e.com/{long}\">kept</a></span> <font face=\"{long}\">too</font></p>");
let sp = spans(&crate::html::parse(&html));
assert!(sp.iter().any(|(t, _)| t.contains("kept")) && sp.iter().any(|(t, _)| t.contains("too")));
assert!(sp.iter().all(|(_, f)| f.font.is_none() && f.link.is_none()), "{:?}", sp.iter().map(|s| &s.0).collect::<Vec<_>>());
let sp = spans(&crate::markdown::parse(&format!("[kept](https://e.com/{long})")));
assert_eq!(sp.len(), 1);
assert_eq!((sp[0].0.as_str(), sp[0].1.link.is_none()), ("kept", true));
}
/// Spans of one link share its target, and so do references to one definition: a long
/// definition and many references no longer copy the destination into each (#3713).
#[test]
fn spans_share_their_link_target() {
let dest = format!("https://e.com/{}", "x".repeat(4000));
let md = format!("{}\n\n[r]: {dest}\n", "[a][r] **[b][r]** ".repeat(10_000));
let sp = spans(&crate::markdown::parse(&md));
let links: Vec<_> = sp.iter().filter_map(|(_, f)| f.link.as_ref()).collect();
assert_eq!(links.len(), 20_000);
assert!(links.iter().all(|l| Arc::ptr_eq(l, links[0])), "one copy of the destination");
let html = format!("<a href=\"{dest}\">{}</a>", "<b>a</b>a".repeat(1000));
let sp = spans(&crate::html::parse(&html));
assert!(sp.len() >= 2000);
let first = sp[0].1.link.clone().unwrap();
assert!(sp.iter().all(|(_, f)| f.link.as_ref().is_some_and(|l| Arc::ptr_eq(l, &first))));
}
/// The document's runs own their link and font strings; past a budget further runs keep their
/// text but lose them, so a flow of shared names can't expand into gigabytes (#3713).
#[test]
fn run_link_and_font_copies_are_budgeted() {
let link: Arc<str> = format!("https://e.com/{}", "x".repeat(model::MAX_LINK - 20)).into();
let mut p = model::Para::default();
for i in 0..20_000 {
let f = model::Fmt { bold: i % 2 == 0, link: Some(link.clone()), ..Default::default() };
p.push_text("a", &f);
}
model::begin_import();
let (doc, warned) = crate::test_log::warnings(|| model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() }));
let paras: Vec<_> = doc.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p) } else { None }).collect();
assert_eq!(paras.iter().map(|p| p.text.len()).sum::<usize>(), 20_000, "all text kept");
let runs: Vec<_> = paras.iter().flat_map(|p| p.runs.iter()).collect();
let kept: usize = runs.iter().filter_map(|r| r.props.link.as_ref()).map(String::len).sum();
assert!(kept > 0 && kept <= 64 << 20, "{kept}");
// The runs that lose theirs are said to, once for the document (review of #3873).
assert_eq!(warned.iter().filter(|w| w.contains("later runs lose them")).count(), 1, "{warned:?}");
}
/// Every limit that leaves a font name, link or style name out says so, once per kind per
/// document rather than once per item (review of #3873; the tracker complains of silent loss).
#[test]
fn dropped_names_and_links_are_logged_once_per_document() {
let long = "x".repeat(model::MAX_LINK + 1);
let html = format!(
"<p><span style=\"font-family:'{long}'\"><a href=\"https://e.com/{long}\">a</a></span> <font face=\"{long}\">b</font> <a href=\"https://e.com/{long}\">c</a></p>"
);
let count = |w: &[String], what: &str| w.iter().filter(|l| l.contains(what)).count();
for _ in 0..2 {
// Each import is its own document: the second is reported too.
let (doc, warned) = crate::test_log::warnings(|| crate::html::import(&html));
assert!(doc.body.len() == 1, "the text is kept");
assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}");
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
}
let (_, warned) = crate::test_log::warnings(|| crate::html::import("<p><a href=\"https://e.com\" style=\"font-family:Georgia\">fine</a></p>"));
assert!(warned.is_empty(), "nothing left out, nothing said: {warned:?}");
let md = format!("[a](https://e.com/{long}) [b](https://e.com/{long})");
let (_, warned) = crate::test_log::warnings(|| crate::markdown::import(&md));
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
let rtf =
format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}{{\\f2 {long};}}}}{{\\stylesheet{{\\s1 {long};}}{{\\s2 {long};}}}}\\f1 a\\f2 b}}");
let (_, warned) = crate::test_log::warnings(|| crate::rtf::import(rtf.as_bytes()).unwrap());
assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}");
assert_eq!(count(&warned, "style names longer"), 1, "{warned:?}");
let tex = format!("\\href{{https://e.com/{long}}}{{a}} \\url{{https://e.com/{long}}}");
let (_, warned) = crate::test_log::warnings(|| crate::latex::import(tex.as_bytes()));
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
}
docs/file-format-parity.md
+4 -2
View File
File diff suppressed because one or more lines are too long