mirror of
https://github.com/storytold/wordcraft.git
synced 2026-10-24 10:27:44 -06:00
Markdown import: brackets no longer turn later links in the paragraph into plain text (#1452) (#1453)
* #1336: Markdown looks up the backtick run closing a code span instead of rescanning the text Code spans and link brackets searched the rest of the text for the run of backticks closing a code span. Every run nothing closes (one per length) cost a scan to the end, and the bracket scan repeated those searches for each `[` over them, outside its look-ahead budget: 2 MB of such runs took 2-18 s. The runs are now indexed by length once per text and found with a binary search; what reads as a code span or a link is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * #1452: Markdown brackets no longer turn later links in the paragraph into plain text The link and image bracket scans share a look-ahead budget per text, so hostile input stays linear, but it was charged more than they read: a `[` nothing closes took the whole budget, and every bracket pair and every link took the rest of the text, as if a destination had been read to the end. An interval like `[0, 1)`, a dozen references in a long paragraph or ten links were enough to make every later link plain text. A scan now pays for the bytes it reads and records the pairs it passes, and the brackets still open at the end of the text, so later brackets are looked up. A destination is read, and paid for, only when `(` follows the `]`; one that parses costs its own length. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rabis7575
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
a4b335f6a3
commit
4d2649738e
@@ -290,6 +290,8 @@ Ranked, to beta (detail and remaining milestones in [docs/roadmap.md](docs/roadm
|
||||
| 2026-10-11 | Word XML files (#548): Word XML Document (Flat OPC `.xml`) opens and saves through the DOCX reader and writer; Word 2003 XML opens (styles, lists, tables, sections, notes, comments, pictures, hyperlinks) |
|
||||
| 2026-10-12 | File › Print selects a system printer, copies, duplex and grayscale, submits PDF jobs through craft-print on a worker and shows spooler errors; browser printing and PDF export remain available |
|
||||
| 2026-10-11 | Font fallback indexes which characters each installed font maps, reading each font file once (#605): text in a script no installed font covers no longer re-reads every system font per distinct character (200 such characters: 13 s → 0.2 s) |
|
||||
| 2026-10-13 | Markdown import of text full of backtick runs that never close is linear (#1336): every code span and link bracket over them searched the rest of the text for a closing run (2 MB in brackets: 17.8 s → 0.06 s) |
|
||||
| 2026-10-13 | Markdown import keeps every link in a paragraph (#1452): a `[` nothing closes, bracketed text that is not a link and each link used up the bracket scans' look-ahead budget as if they had read the rest of the paragraph, so later links were plain text; a scan now costs what it reads, and brackets it has passed are looked up |
|
||||
| 2026-10-11 | Find and Replace special characters (`^p`, `^t`, `^#`, `^&` …) and Use wildcards (ranges, counts, word anchors, groups with `\1` in Replace); `^p` finds and splits paragraphs; Special ▾ menu and search options in the dialog (#543) |
|
||||
| 2026-10-11 | Typing parity (#546): `+---+` Enter makes a table, `> ` `-> ` `=> ` arrow bullets, smileys inside words (`ok:)`), Undo right after a border line brings the characters back |
|
||||
| 2026-10-11 | Windows: DirectX 12 shaders compile with FXC, built into Windows, so the app no longer quits at launch with "Parent device is lost" when no matching `dxcompiler.dll` + `dxil.dll` pair is around (#603) |
|
||||
@@ -485,6 +487,8 @@ Ranked, to beta (detail and remaining milestones in [docs/roadmap.md](docs/roadm
|
||||
| 2026-10-11 | trivial | Quick Access Toolbar KeyTips and persisted enable/disable setting (#620) |
|
||||
| 2026-10-11 | trivial | Progress log: mail-merge recipients from .xlsx/.xlsm/.ods (#334) |
|
||||
| 2026-10-11 | trivial | Insert › Icons landed (#309): catalog 402/434 → 403/434 (92.9%) |
|
||||
| 2026-10-13 | trivial | Progress log: Markdown backtick runs read in linear time (#1336) |
|
||||
| 2026-10-13 | trivial | Progress log: Markdown brackets no longer turn later links into text (#1452) |
|
||||
| 2026-10-11 | trivial | Enclose Characters (#297): catalog 403/434 (92.9%) |
|
||||
| 2026-10-12 | minor | Progress log: opaque custom XML store preservation, with synthetic package regressions and explicit limits |
|
||||
| 2026-10-12 | minor | Preserve imported TOC ranges and retain #538 refresh/undo/save coverage on current main; readiness unchanged |
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
//!
|
||||
//! Export writes the same constructs; pictures become `` placeholders.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use wordcraft_doc::{Align, Document};
|
||||
|
||||
use crate::html::decode_entity;
|
||||
@@ -556,10 +558,11 @@ fn push_text(nodes: &mut Vec<Node>, s: &str, f: &Fmt) {
|
||||
}
|
||||
|
||||
/// The closing `]` matching the `[` at byte `open` (escapes and code spans respected). Gives up
|
||||
/// after looking `max` bytes ahead.
|
||||
fn matching_bracket(s: &str, open: usize, max: usize) -> Option<usize> {
|
||||
/// after looking `max` bytes ahead. Records in `known` every bracket pair it passes, and every `[`
|
||||
/// still open at the end of the text: a scan from one of those would take the same path.
|
||||
fn matching_bracket(s: &str, open: usize, max: usize, ticks: &Backticks, known: &mut HashMap<usize, Option<usize>>) -> Option<usize> {
|
||||
let b = s.as_bytes();
|
||||
let mut depth = 0usize;
|
||||
let mut opens: Vec<usize> = Vec::new();
|
||||
let mut i = open;
|
||||
while let Some(&c) = b.get(i) {
|
||||
if i - open > max {
|
||||
@@ -569,16 +572,18 @@ fn matching_bracket(s: &str, open: usize, max: usize) -> Option<usize> {
|
||||
b'\\' => i += 1,
|
||||
b'`' => {
|
||||
let n = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1);
|
||||
if let Some(end) = find_backticks(s, i + n, n) {
|
||||
if let Some(end) = ticks.find(i + n, n) {
|
||||
i = end + n - 1;
|
||||
} else {
|
||||
i += n - 1;
|
||||
}
|
||||
}
|
||||
b'[' => depth += 1,
|
||||
b'[' => opens.push(i),
|
||||
b']' => {
|
||||
depth = depth.saturating_sub(1);
|
||||
if depth == 0 {
|
||||
if let Some(o) = opens.pop() {
|
||||
known.insert(o, Some(i));
|
||||
}
|
||||
if opens.is_empty() {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
@@ -586,25 +591,39 @@ fn matching_bracket(s: &str, open: usize, max: usize) -> Option<usize> {
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
for o in opens {
|
||||
known.insert(o, None);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Byte index of a run of exactly `n` backticks at or after `from`.
|
||||
fn find_backticks(s: &str, from: usize, n: usize) -> Option<usize> {
|
||||
let b = s.as_bytes();
|
||||
let mut i = from;
|
||||
while let Some(&c) = b.get(i) {
|
||||
if c == b'`' {
|
||||
let k = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1);
|
||||
if k == n {
|
||||
return Some(i);
|
||||
/// Where a text's runs of backticks start, by run length. Finding the run that closes a code span
|
||||
/// is then a lookup: scanning the rest of the text for it, again for every run nothing closes
|
||||
/// (each of a different length) and from every link bracket over them, was superlinear.
|
||||
struct Backticks(HashMap<usize, Vec<usize>>);
|
||||
|
||||
impl Backticks {
|
||||
fn new(s: &str) -> Self {
|
||||
let b = s.as_bytes();
|
||||
let mut runs: HashMap<usize, Vec<usize>> = HashMap::new();
|
||||
let mut i = 0;
|
||||
while let Some(&c) = b.get(i) {
|
||||
if c == b'`' {
|
||||
let k = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1);
|
||||
runs.entry(k).or_default().push(i);
|
||||
i += k;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
i += k;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
Self(runs)
|
||||
}
|
||||
|
||||
/// Byte index of a run of exactly `n` backticks at or after `from` (which is not inside a run).
|
||||
fn find(&self, from: usize, n: usize) -> Option<usize> {
|
||||
let starts = self.0.get(&n)?;
|
||||
starts.get(starts.partition_point(|&p| p < from)).copied()
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Parse `(dest "title")` at byte `at` (which holds `(`): (destination, index after `)`).
|
||||
@@ -706,8 +725,11 @@ fn tokenize(s: &str, base: &Fmt, depth: usize) -> Vec<Node> {
|
||||
let mut nodes: Vec<Node> = Vec::new();
|
||||
// Bytes the link and image scans below may look ahead in total. Each unmatched `[` or `![(`
|
||||
// would otherwise rescan the rest of the text, which is quadratic on hostile input; once the
|
||||
// budget is gone, further brackets are plain text.
|
||||
// budget is gone, further brackets are plain text. A scan pays for what it reads; brackets an
|
||||
// earlier scan passed are looked up in `known` instead.
|
||||
let mut scan_budget = s.len().saturating_mul(8).saturating_add(8192);
|
||||
let ticks = Backticks::new(s);
|
||||
let mut known: HashMap<usize, Option<usize>> = HashMap::new();
|
||||
let b = s.as_bytes();
|
||||
let mut i = 0usize;
|
||||
let mut text_start = 0usize;
|
||||
@@ -737,7 +759,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize) -> Vec<Node> {
|
||||
b'`' => {
|
||||
flush!();
|
||||
let n = b.get(i..).map(|r| r.iter().take_while(|x| **x == b'`').count()).unwrap_or(1);
|
||||
match find_backticks(s, i + n, n) {
|
||||
match ticks.find(i + n, n) {
|
||||
Some(end) => {
|
||||
let mut code = s.get(i + n..end).unwrap_or("").replace('\n', " ");
|
||||
if code.len() >= 2 && code.starts_with(' ') && code.ends_with(' ') && !code.trim().is_empty() {
|
||||
@@ -779,13 +801,21 @@ fn tokenize(s: &str, base: &Fmt, depth: usize) -> Vec<Node> {
|
||||
b'!' | b'[' if depth < MAX_NEST => {
|
||||
let img = c == b'!';
|
||||
let open = if img { i + 1 } else { i };
|
||||
let parsed = if b.get(open) == Some(&b'[') && scan_budget > 0 {
|
||||
let found = matching_bracket(s, open, scan_budget);
|
||||
scan_budget = scan_budget.saturating_sub(found.map_or(scan_budget, |close| close - open));
|
||||
found.and_then(|close| {
|
||||
let parsed = if b.get(open) == Some(&b'[') {
|
||||
let found = match known.get(&open).copied() {
|
||||
Some(found) => found,
|
||||
None if scan_budget > 0 => {
|
||||
let found = matching_bracket(s, open, scan_budget, &ticks, &mut known);
|
||||
scan_budget = scan_budget.saturating_sub(found.unwrap_or(s.len()).saturating_sub(open));
|
||||
found
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
found.filter(|close| b.get(close + 1) == Some(&b'(') && scan_budget > 0).and_then(|close| {
|
||||
// A destination that does not parse may have been read to the end of the text.
|
||||
scan_budget = scan_budget.saturating_sub(s.len().saturating_sub(close));
|
||||
let (dest, end) = link_dest(s, close + 1)?;
|
||||
let dest = link_dest(s, close + 1);
|
||||
scan_budget = scan_budget.saturating_sub(dest.as_ref().map_or(s.len(), |d| d.1).saturating_sub(close));
|
||||
let (dest, end) = dest?;
|
||||
Some((s.get(open + 1..close)?.to_string(), dest, end))
|
||||
})
|
||||
} else {
|
||||
|
||||
@@ -618,6 +618,65 @@ fn hostile_importer_input_takes_linear_time() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Backtick runs that never close, each a different length, sent every code span and every link
|
||||
/// bracket scan over them looking for a closing run through the rest of the text: superlinear
|
||||
/// time on hostile Markdown.
|
||||
#[test]
|
||||
fn unmatched_backtick_runs_take_linear_time() {
|
||||
use std::time::{Duration, Instant};
|
||||
// Runs of 1, 2, … 2000 backticks (2 MB), none of them closed.
|
||||
let runs = (1..=2000).map(|n| "`".repeat(n)).collect::<Vec<_>>().join(" ");
|
||||
let cases = [runs.clone(), format!("{}{runs}{}", "[".repeat(8), "]".repeat(8)), format!("[{runs}")];
|
||||
for text in cases {
|
||||
let t = Instant::now();
|
||||
let d = import("md", text.as_bytes()).unwrap().unwrap();
|
||||
let plain = d.plain_text(Default::default());
|
||||
let elapsed = t.elapsed();
|
||||
eprintln!("{:?}... took {elapsed:?}", &text[..4]);
|
||||
assert!(elapsed < Duration::from_secs(5), "{:?}... took {elapsed:?}", &text[..4]);
|
||||
assert_eq!(plain.matches('`').count(), text.matches('`').count(), "every backtick stays text");
|
||||
}
|
||||
}
|
||||
|
||||
/// Code spans and brackets around backticks read as before the backtick runs were indexed.
|
||||
#[test]
|
||||
fn code_spans_and_links_around_backticks_read_as_before() {
|
||||
// An escaped backtick before a span, a span holding a shorter run, a run nothing closes, a `]`
|
||||
// inside a code span in link text, and link text with a backtick that nothing closes.
|
||||
let flow = crate::markdown::parse("\\``e` `` a ` b `` then ``` and `c` [x `]` y](http://a.org) [p ` q](http://b.org) ``f");
|
||||
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
|
||||
let spans: Vec<(String, bool, Option<String>)> =
|
||||
p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.clone())) } else { None }).collect();
|
||||
let code: Vec<&str> = spans.iter().filter(|s| s.1).map(|s| s.0.as_str()).collect();
|
||||
assert_eq!(code, ["e", "a ` b", "c", "]"], "{spans:?}");
|
||||
let linked = |url: &str| spans.iter().filter(|s| s.2.as_deref() == Some(url)).map(|s| s.0.as_str()).collect::<String>();
|
||||
assert_eq!(linked("http://a.org"), "x ] y");
|
||||
assert_eq!(linked("http://b.org"), "p ` q");
|
||||
assert_eq!(p.text(), "`e a ` b then ``` and c x ] y p ` q ``f");
|
||||
}
|
||||
|
||||
/// A `[` nothing closes, bracketed text that is not a link and links themselves used up the bracket
|
||||
/// scans' budget as if each had read the rest of the text: later links in the paragraph were plain
|
||||
/// text.
|
||||
#[test]
|
||||
fn brackets_do_not_turn_later_links_into_text() {
|
||||
let links = |md: &str| {
|
||||
let flow = crate::markdown::parse(md);
|
||||
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
|
||||
let mut urls: Vec<String> = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.clone() } else { None }).collect();
|
||||
urls.dedup();
|
||||
urls
|
||||
};
|
||||
// An interval nothing closes, before a link.
|
||||
assert_eq!(links("Intervals [0, 1) and [docs](http://x.org)"), ["http://x.org"]);
|
||||
// References early in a long paragraph, a link at its end.
|
||||
let refs = (1..=12).map(|n| format!("[{n}]")).collect::<Vec<_>>().join(" ");
|
||||
assert_eq!(links(&format!("{refs} {}[docs](http://x.org)", "word ".repeat(4000))), ["http://x.org"]);
|
||||
// Many links through one paragraph.
|
||||
let urls: Vec<String> = (0..60).map(|n| format!("http://x.org/{n}")).collect();
|
||||
assert_eq!(links(&urls.iter().map(|u| format!("[link]({u}) {}", "word ".repeat(20))).collect::<String>()), urls);
|
||||
}
|
||||
|
||||
/// The bounded scans must not change how ordinary links, images, tags and entities read.
|
||||
#[test]
|
||||
fn bounded_scans_keep_ordinary_markdown_and_entities() {
|
||||
|
||||
Reference in New Issue
Block a user