mirror of
https://github.com/storytold/designcraft.git
synced 2026-10-29 12:46:53 -06:00
Merge #316: GREP: the dialect's classes and metacharacters (\l \u \h, ~ codes, lookaround) in GREP styles and Find/Change
Conflict: crates/compose/src/tests.rs: tests only, both (#310's tight word spacing test, #316's GREP style test) #316's overlay test calls overlays() with the drop_end argument next added (#83).
This commit is contained in:
aiongg
Generated
+12
-2
@@ -922,6 +922,7 @@ dependencies = [
|
||||
"designcraft-doc",
|
||||
"designcraft-fonts",
|
||||
"designcraft-geom",
|
||||
"fancy-regex",
|
||||
"hypher",
|
||||
"log",
|
||||
"miniz_oxide 0.8.9",
|
||||
@@ -969,7 +970,6 @@ dependencies = [
|
||||
"kurbo",
|
||||
"log",
|
||||
"quick-xml 0.38.4",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
@@ -1137,7 +1137,6 @@ dependencies = [
|
||||
"egui",
|
||||
"egui_kittest",
|
||||
"log",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"toml",
|
||||
@@ -1503,6 +1502,17 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912"
|
||||
dependencies = [
|
||||
"bit-set 0.8.0",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
|
||||
@@ -44,6 +44,7 @@ unicode-bidi-mirroring = "0.4"
|
||||
unicode-segmentation = "1"
|
||||
caseless = "0.2"
|
||||
regex = { version = "1", default-features = false, features = ["std", "unicode", "perf"] }
|
||||
fancy-regex = "0.19.2"
|
||||
tempfile = "3"
|
||||
|
||||
# text / rendering / formats
|
||||
|
||||
@@ -18,6 +18,7 @@ designcraft-doc = { workspace = true }
|
||||
designcraft-fonts = { workspace = true }
|
||||
log = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
fancy-regex = { workspace = true }
|
||||
miniz_oxide = "0.8"
|
||||
hypher = { version = "0.1.8", default-features = false, features = ["alloc", "spanish"] }
|
||||
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
//! The GREP dialect of page-layout documents (Perl-style regular expressions with letter-class
|
||||
//! escapes and `~` metacharacters), compiled for our story encoding.
|
||||
//!
|
||||
//! [`translate`] rewrites the dialect into the syntax `fancy_regex` reads; [`Grep::new`] compiles
|
||||
//! it. GREP styles, Find/Change and the style dialogs all go through here.
|
||||
//!
|
||||
//! - `\l` `\u` (and `\L` `\U`) are Unicode lowercase and uppercase letters, `\h` `\H` horizontal
|
||||
//! whitespace, `\v` `\V` vertical whitespace, `\R` any line break, `\N` anything but a
|
||||
//! paragraph end.
|
||||
//! - `\r` is the paragraph end (`\n` in a story), `\n` the forced line break (U+2028).
|
||||
//! - `~` codes name special characters (`~e` ellipsis, `~=` en dash, `~S` nonbreaking space…).
|
||||
//! - POSIX classes (`[[:alpha:]]`) are Unicode-aware.
|
||||
//! - Lookahead, lookbehind, atomic groups, possessive quantifiers, backreferences and `\K` are
|
||||
//! supported. Backtracking is capped ([`BACKTRACK_LIMIT`]), so a pathological pattern stops
|
||||
//! with an error instead of hanging.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
|
||||
use designcraft_doc::story;
|
||||
|
||||
/// Longest pattern accepted, in bytes.
|
||||
pub const MAX_PATTERN: usize = 4096;
|
||||
/// Backtracking steps one search may take before it gives up.
|
||||
pub const BACKTRACK_LIMIT: usize = 200_000;
|
||||
/// Compiled program size cap (bytes) for the underlying regex engine.
|
||||
const SIZE_LIMIT: usize = 4 << 20;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct GrepError(pub String);
|
||||
|
||||
impl fmt::Display for GrepError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for GrepError {}
|
||||
|
||||
impl From<GrepError> for String {
|
||||
fn from(e: GrepError) -> String {
|
||||
e.0
|
||||
}
|
||||
}
|
||||
|
||||
fn err(msg: impl Into<String>) -> GrepError {
|
||||
GrepError(msg.into())
|
||||
}
|
||||
|
||||
/// Characters a `~` code stands for. Several characters means "any of these".
|
||||
fn tilde(c: char) -> Option<&'static [char]> {
|
||||
Some(match c {
|
||||
'b' => &['\n'],
|
||||
'#' => &[story::PAGE_NUMBER, story::NEXT_PAGE_NUMBER, story::PREV_PAGE_NUMBER],
|
||||
'N' => &[story::PAGE_NUMBER],
|
||||
'X' => &[story::NEXT_PAGE_NUMBER],
|
||||
'V' => &[story::PREV_PAGE_NUMBER],
|
||||
'x' => &[story::SECTION_MARKER],
|
||||
'M' => &[story::COLUMN_BREAK],
|
||||
'R' => &[story::FRAME_BREAK],
|
||||
'P' => &[story::PAGE_BREAK],
|
||||
'L' => &[story::ODD_PAGE_BREAK],
|
||||
'E' => &[story::EVEN_PAGE_BREAK],
|
||||
'i' => &[story::INDENT_HERE],
|
||||
'y' => &[story::RIGHT_INDENT_TAB],
|
||||
'n' => &[story::FORCED_LINE_BREAK],
|
||||
'k' => &['\u{200B}'],
|
||||
'j' => &['\u{200C}'],
|
||||
'=' => &['\u{2013}'],
|
||||
'_' => &['\u{2014}'],
|
||||
'-' => &['\u{AD}'],
|
||||
'~' => &['\u{2011}'],
|
||||
'S' => &['\u{A0}'],
|
||||
's' => &['\u{202F}'],
|
||||
'f' => &['\u{2001}'],
|
||||
'm' => &['\u{2003}'],
|
||||
'>' => &['\u{2002}'],
|
||||
'3' => &['\u{2004}'],
|
||||
'4' => &['\u{2005}'],
|
||||
'%' => &['\u{2006}'],
|
||||
'/' => &['\u{2007}'],
|
||||
'.' => &['\u{2008}'],
|
||||
'<' => &['\u{2009}'],
|
||||
'|' => &['\u{200A}'],
|
||||
'e' => &['\u{2026}'],
|
||||
'8' => &['\u{2022}'],
|
||||
'2' => &['\u{A9}'],
|
||||
'r' => &['\u{AE}'],
|
||||
'd' => &['\u{2122}'],
|
||||
'6' => &['\u{A7}'],
|
||||
'7' => &['\u{B6}'],
|
||||
'{' => &['\u{201C}'],
|
||||
'}' => &['\u{201D}'],
|
||||
'[' => &['\u{2018}'],
|
||||
']' => &['\u{2019}'],
|
||||
'"' => &['"', '\u{201C}', '\u{201D}'],
|
||||
'\'' => &['\'', '\u{2018}', '\u{2019}'],
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
const H_SPACE: &str = r"\t\p{Zs}";
|
||||
const V_SPACE: &str = r"\n\x0B\f\r\x{85}\x{2028}\x{2029}";
|
||||
|
||||
/// Class members as they appear inside `[...]`, or a whole class / group outside one.
|
||||
enum Piece {
|
||||
/// Valid both inside and outside a class as is.
|
||||
Atom(&'static str),
|
||||
/// Class members: wrapped in `[...]` outside a class, spliced in inside one.
|
||||
Members(&'static str),
|
||||
/// Negated members: `[^...]` (nested when inside a class).
|
||||
NotMembers(&'static str),
|
||||
/// Only valid outside a class.
|
||||
Outside(&'static str),
|
||||
}
|
||||
|
||||
/// The translation of `\c`, or None to keep the escape as is.
|
||||
fn escape(c: char) -> Option<Piece> {
|
||||
Some(match c {
|
||||
'l' => Piece::Atom(r"\p{Ll}"),
|
||||
'u' => Piece::Atom(r"\p{Lu}"),
|
||||
'L' => Piece::Atom(r"\P{Ll}"),
|
||||
'U' => Piece::Atom(r"\P{Lu}"),
|
||||
'h' => Piece::Members(H_SPACE),
|
||||
'H' => Piece::NotMembers(H_SPACE),
|
||||
'v' => Piece::Members(V_SPACE),
|
||||
'V' => Piece::NotMembers(V_SPACE),
|
||||
'r' => Piece::Atom(r"\n"),
|
||||
'n' => Piece::Atom(r"\x{2028}"),
|
||||
'N' => Piece::Outside(r"[^\n]"),
|
||||
'R' => Piece::Outside(r"(?:\r\n|[\n\x0B\f\r\x{85}\x{2028}\x{2029}])"),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Members of a POSIX class `[:name:]` (Unicode-aware, unlike the regex crate's ASCII ones).
|
||||
fn posix(name: &str) -> Option<&'static str> {
|
||||
Some(match name {
|
||||
"alpha" => r"\p{Alphabetic}",
|
||||
"lower" => r"\p{Ll}",
|
||||
"upper" => r"\p{Lu}",
|
||||
"digit" => r"\p{Nd}",
|
||||
"alnum" => r"\p{Alphabetic}\p{Nd}",
|
||||
"word" => r"\w",
|
||||
"space" => r"\s",
|
||||
"blank" => H_SPACE,
|
||||
"punct" => r"\p{P}",
|
||||
"cntrl" => r"\p{Cc}",
|
||||
"xdigit" => r"0-9A-Fa-f",
|
||||
"graph" => r"^\s\p{Cc}\p{Cn}",
|
||||
"print" => r"^\p{Cc}\p{Cn}",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn push_char(out: &mut String, c: char) {
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(out, r"\x{{{:X}}}", c as u32);
|
||||
}
|
||||
|
||||
/// Rewrite a GREP-dialect pattern into `fancy_regex` syntax for our story encoding.
|
||||
pub fn translate(pattern: &str) -> Result<String, GrepError> {
|
||||
if pattern.len() > MAX_PATTERN {
|
||||
return Err(err(format!("GREP pattern longer than {MAX_PATTERN} bytes")));
|
||||
}
|
||||
let chars: Vec<char> = pattern.chars().collect();
|
||||
let mut out = String::with_capacity(pattern.len() + 16);
|
||||
let mut i = 0;
|
||||
let mut in_class = false;
|
||||
// The previous class member was a plain `-`: a second one would read as set difference.
|
||||
let mut dash = false;
|
||||
while let Some(&c) = chars.get(i) {
|
||||
i += 1;
|
||||
let was_dash = std::mem::take(&mut dash);
|
||||
match c {
|
||||
'\\' => {
|
||||
let Some(&n) = chars.get(i) else { return Err(err("GREP pattern ends with a backslash")) };
|
||||
i += 1;
|
||||
match escape(n) {
|
||||
None => {
|
||||
out.push('\\');
|
||||
out.push(n);
|
||||
}
|
||||
Some(Piece::Atom(s)) => out.push_str(s),
|
||||
Some(Piece::Members(s)) if in_class => out.push_str(s),
|
||||
Some(Piece::Members(s)) => {
|
||||
out.push('[');
|
||||
out.push_str(s);
|
||||
out.push(']');
|
||||
}
|
||||
Some(Piece::NotMembers(s)) => {
|
||||
out.push_str("[^");
|
||||
out.push_str(s);
|
||||
out.push(']');
|
||||
}
|
||||
Some(Piece::Outside(_)) if in_class => {
|
||||
return Err(err(format!(r"GREP: \{n} can't be used inside [...]")));
|
||||
}
|
||||
Some(Piece::Outside(s)) => out.push_str(s),
|
||||
}
|
||||
}
|
||||
'~' => match chars.get(i).and_then(|&n| tilde(n)) {
|
||||
Some(set) => {
|
||||
i += 1;
|
||||
let wrap = set.len() > 1 && !in_class;
|
||||
if wrap {
|
||||
out.push('[');
|
||||
}
|
||||
for &ch in set {
|
||||
push_char(&mut out, ch);
|
||||
}
|
||||
if wrap {
|
||||
out.push(']');
|
||||
}
|
||||
}
|
||||
None => out.push_str(r"\~"),
|
||||
},
|
||||
'[' if !in_class => {
|
||||
in_class = true;
|
||||
out.push('[');
|
||||
if chars.get(i) == Some(&'^') {
|
||||
out.push('^');
|
||||
i += 1;
|
||||
}
|
||||
// A `]` first in a class is a literal.
|
||||
if chars.get(i) == Some(&']') {
|
||||
out.push_str(r"\]");
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
'[' => {
|
||||
if chars.get(i) == Some(&':') {
|
||||
let rest = chars.get(i + 1..).unwrap_or(&[]);
|
||||
let close = rest.windows(2).position(|w| w == [':', ']']).ok_or_else(|| err("GREP: unclosed [:class:]"))?;
|
||||
let raw: String = rest.get(..close).unwrap_or(&[]).iter().collect();
|
||||
let (neg, name) = match raw.strip_prefix('^') {
|
||||
Some(n) => (true, n),
|
||||
None => (false, raw.as_str()),
|
||||
};
|
||||
let members = posix(name).ok_or_else(|| err(format!("GREP: unknown class [:{raw}:]")))?;
|
||||
// Nested so negation and the `^` in graph/print stay local to this class.
|
||||
out.push('[');
|
||||
if neg {
|
||||
out.push('^');
|
||||
out.push('[');
|
||||
out.push_str(members);
|
||||
out.push(']');
|
||||
} else {
|
||||
out.push_str(members);
|
||||
}
|
||||
out.push(']');
|
||||
i += 1 + close + 2;
|
||||
} else {
|
||||
// A bare `[` inside a class is a literal in this dialect, a nested class in ours.
|
||||
out.push_str(r"\[");
|
||||
}
|
||||
}
|
||||
']' if in_class => {
|
||||
in_class = false;
|
||||
out.push(']');
|
||||
}
|
||||
'&' if in_class => out.push_str(r"\&"),
|
||||
'-' if in_class => {
|
||||
if was_dash {
|
||||
out.push_str(r"\-");
|
||||
} else {
|
||||
out.push('-');
|
||||
dash = true;
|
||||
}
|
||||
}
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Rewrite a GREP change-to string into an expansion template: `$0`…`$9` are groups; `\r` `\n`
|
||||
/// `\t`, `~` codes, any other `$` and `\` + any other character are literal text.
|
||||
pub fn translate_replacement(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut it = s.chars().peekable();
|
||||
let lit = |out: &mut String, c: char| {
|
||||
if c == '$' {
|
||||
out.push_str("$$");
|
||||
} else {
|
||||
out.push(c);
|
||||
}
|
||||
};
|
||||
while let Some(c) = it.next() {
|
||||
match c {
|
||||
'\\' => match it.next() {
|
||||
Some('r') => out.push('\n'),
|
||||
Some('n') => out.push(story::FORCED_LINE_BREAK),
|
||||
Some('t') => out.push('\t'),
|
||||
Some(n) => lit(&mut out, n),
|
||||
None => out.push('\\'),
|
||||
},
|
||||
'~' => match it.peek().and_then(|&n| tilde(n)).and_then(|set| set.first()) {
|
||||
Some(&ch) => {
|
||||
it.next();
|
||||
lit(&mut out, ch);
|
||||
}
|
||||
None => out.push('~'),
|
||||
},
|
||||
// `$1abc` is group 1 then "abc"; a `$` before anything but a digit is literal.
|
||||
'$' => match it.peek().copied().filter(char::is_ascii_digit) {
|
||||
Some(d) => {
|
||||
it.next();
|
||||
out.push_str("${");
|
||||
out.push(d);
|
||||
out.push('}');
|
||||
}
|
||||
None => out.push_str("$$"),
|
||||
},
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Matching options.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct GrepOptions {
|
||||
pub case_insensitive: bool,
|
||||
/// `^` and `$` match at paragraph starts and ends, not only at the ends of the text.
|
||||
pub multi_line: bool,
|
||||
pub whole_word: bool,
|
||||
}
|
||||
|
||||
/// A compiled GREP pattern.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Grep {
|
||||
re: fancy_regex::Regex,
|
||||
}
|
||||
|
||||
impl Grep {
|
||||
pub fn new(pattern: &str, opts: GrepOptions) -> Result<Grep, GrepError> {
|
||||
Grep::build(translate(pattern)?, opts)
|
||||
}
|
||||
|
||||
/// A pattern matching `text` literally.
|
||||
pub fn literal(text: &str, opts: GrepOptions) -> Result<Grep, GrepError> {
|
||||
if text.len() > MAX_PATTERN {
|
||||
return Err(err(format!("search text longer than {MAX_PATTERN} bytes")));
|
||||
}
|
||||
Grep::build(regex::escape(text), opts)
|
||||
}
|
||||
|
||||
fn build(mut pat: String, opts: GrepOptions) -> Result<Grep, GrepError> {
|
||||
if opts.whole_word {
|
||||
pat = format!(r"\b(?:{pat})\b");
|
||||
}
|
||||
let flags = match (opts.case_insensitive, opts.multi_line) {
|
||||
(true, true) => "(?im)",
|
||||
(true, false) => "(?i)",
|
||||
(false, true) => "(?m)",
|
||||
(false, false) => "",
|
||||
};
|
||||
let re = fancy_regex::RegexBuilder::new(&format!("{flags}{pat}"))
|
||||
.backtrack_limit(BACKTRACK_LIMIT)
|
||||
.delegate_size_limit(SIZE_LIMIT)
|
||||
.delegate_dfa_size_limit(SIZE_LIMIT)
|
||||
.build()
|
||||
.map_err(|e| err(format!("bad GREP: {e}")))?;
|
||||
Ok(Grep { re })
|
||||
}
|
||||
|
||||
/// The first match starting at or after byte `start`.
|
||||
pub fn find_at(&self, text: &str, start: usize) -> Result<Option<Range<usize>>, GrepError> {
|
||||
if start > text.len() || !text.is_char_boundary(start) {
|
||||
return Ok(None);
|
||||
}
|
||||
self.re.find_from_pos(text, start).map(|m| m.map(|m| m.range())).map_err(run_err)
|
||||
}
|
||||
|
||||
/// The first `limit` non-overlapping matches in `text`.
|
||||
pub fn find_all(&self, text: &str, limit: usize) -> Result<Vec<Range<usize>>, GrepError> {
|
||||
self.re.find_iter(text).take(limit).map(|m| m.map(|m| m.range()).map_err(run_err)).collect()
|
||||
}
|
||||
|
||||
/// Matches in `text` and their replacements, `template` expanded per match (`$1`…; see
|
||||
/// [`translate_replacement`]). Stops after the first match when `first_only`.
|
||||
pub fn replacements(&self, text: &str, template: &str, first_only: bool) -> Result<Vec<(Range<usize>, String)>, GrepError> {
|
||||
let exp = fancy_regex::Expander::default();
|
||||
let mut out = Vec::new();
|
||||
for caps in self.re.captures_iter(text) {
|
||||
let caps = caps.map_err(run_err)?;
|
||||
let Some(m) = caps.get(0) else { continue };
|
||||
out.push((m.range(), exp.expansion(template, &caps)));
|
||||
if first_only {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn run_err(e: fancy_regex::Error) -> GrepError {
|
||||
err(format!("GREP search stopped: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn grep(p: &str) -> Grep {
|
||||
Grep::new(p, GrepOptions::default()).unwrap()
|
||||
}
|
||||
|
||||
fn found<'a>(p: &str, text: &'a str) -> Vec<&'a str> {
|
||||
grep(p).find_all(text, usize::MAX).unwrap().into_iter().map(|r| &text[r]).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translates_dialect_tokens() {
|
||||
let t = |p: &str| translate(p).unwrap();
|
||||
assert_eq!(t(r"\l\u\L\U"), r"\p{Ll}\p{Lu}\P{Ll}\P{Lu}");
|
||||
assert_eq!(t(r"\h"), r"[\t\p{Zs}]");
|
||||
assert_eq!(t(r"[\h,]"), r"[\t\p{Zs},]");
|
||||
assert_eq!(t(r"\H"), r"[^\t\p{Zs}]");
|
||||
assert_eq!(t(r"\v"), r"[\n\x0B\f\r\x{85}\x{2028}\x{2029}]");
|
||||
assert_eq!(t(r"\r\n"), r"\n\x{2028}");
|
||||
assert_eq!(t(r"~e~=~_~S~<~>~m~%~/"), r"\x{2026}\x{2013}\x{2014}\x{A0}\x{2009}\x{2002}\x{2003}\x{2006}\x{2007}");
|
||||
assert_eq!(t(r#"~""#), r"[\x{22}\x{201C}\x{201D}]");
|
||||
assert_eq!(t("a~q"), r"a\~q", "an unknown ~ code is a literal tilde");
|
||||
// Escaped backslashes and other escapes stay put.
|
||||
assert_eq!(t(r"\\l\\h"), r"\\l\\h");
|
||||
assert_eq!(t(r"\d\w\s\b\<\>\x{41}\p{Lu}"), r"\d\w\s\b\<\>\x{41}\p{Lu}");
|
||||
// Character classes: leading `]`, bare `[`, `&&`, `--`, POSIX classes.
|
||||
assert_eq!(t(r"[]a]"), r"[\]a]");
|
||||
assert_eq!(t(r"[^]a]"), r"[^\]a]");
|
||||
assert_eq!(t(r"[a[b]"), r"[a\[b]");
|
||||
assert_eq!(t(r"[a&&b]"), r"[a\&\&b]");
|
||||
assert_eq!(t(r"[+--]"), r"[+-\-]");
|
||||
assert_eq!(t(r"[[:alpha:]]"), r"[[\p{Alphabetic}]]");
|
||||
assert_eq!(t(r"[[:^digit:]x]"), r"[[^[\p{Nd}]]x]");
|
||||
assert!(translate(r"[[:bogus:]]").is_err());
|
||||
assert!(translate(r"[\R]").is_err());
|
||||
assert!(translate("a\\").is_err());
|
||||
assert!(translate(&"a".repeat(MAX_PATTERN + 1)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_like_the_dialect() {
|
||||
// Russian one- and two-letter words followed by a space (a no-break GREP style).
|
||||
assert_eq!(found(r"\b\l{1,2}\h", "Я и он в доме"), ["и ", "он ", "в "]);
|
||||
assert_eq!(found(r"\u\l+", "Hello World ok"), ["Hello", "World"]);
|
||||
assert_eq!(found(r"[[:upper:]]+", "abc ÉTÉ"), ["ÉTÉ"]);
|
||||
assert_eq!(found(r"[]x]+", "a]x]b"), ["]x]"]);
|
||||
assert_eq!(found(r"\d~=\d", "1\u{2013}2 3-4"), ["1\u{2013}2"]);
|
||||
assert_eq!(found(r"\<\w", "ab cd"), ["a", "c"]);
|
||||
assert_eq!(found(r"\\l", r"a\lb"), [r"\l"]);
|
||||
// Lookaround, atomic groups, possessive quantifiers, \K.
|
||||
assert_eq!(found(r"(?<=\d)\h(?=\l)", "5 kg 5 K"), [" "]);
|
||||
assert_eq!(found(r"(?>a+)b", "aab"), ["aab"]);
|
||||
assert_eq!(found(r"a++b", "aab"), ["aab"]);
|
||||
assert_eq!(found(r"\d+\K%", "50% off"), ["%"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn options_and_replacements() {
|
||||
let g = Grep::new(r"\u\l+", GrepOptions { case_insensitive: false, multi_line: true, whole_word: true }).unwrap();
|
||||
assert_eq!(g.find_at("Ab Cd", 1).unwrap(), Some(3..5));
|
||||
assert_eq!(g.find_at("Ab Cd", 99).unwrap(), None);
|
||||
let g = Grep::literal("a.b", GrepOptions { case_insensitive: true, ..Default::default() }).unwrap();
|
||||
assert_eq!(g.find_all("axb A.B a.b", 1).unwrap().first(), Some(&(4..7)));
|
||||
let g = grep(r"(\d+)\h(\l+)");
|
||||
let tpl = translate_replacement(r"$2~S$1x\r\$$");
|
||||
assert_eq!(g.replacements("5 kg", &tpl, false).unwrap(), [(0..4, "kg\u{A0}5x\n$$".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catastrophic_backtracking_stops() {
|
||||
// Exponential in the backtracking engine (the backreference forces it): without the limit
|
||||
// this runs for hours.
|
||||
let g = grep(r"(a|aa)+\1x");
|
||||
let text = "a".repeat(64);
|
||||
let t = std::time::Instant::now();
|
||||
assert!(g.find_all(&text, usize::MAX).is_err());
|
||||
assert!(t.elapsed().as_secs() < 5);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
mod bidi;
|
||||
pub mod breaker;
|
||||
mod cache;
|
||||
pub mod grep;
|
||||
pub mod hyphen;
|
||||
mod mojikumi;
|
||||
mod notes;
|
||||
|
||||
@@ -7,8 +7,11 @@ use std::ops::Range;
|
||||
|
||||
use designcraft_doc::{GrepStyle, NestedStyle, NestedUntil};
|
||||
|
||||
use crate::grep::{Grep, GrepOptions};
|
||||
|
||||
thread_local! {
|
||||
static PATTERNS: RefCell<HashMap<String, Option<regex::Regex>>> = RefCell::new(HashMap::new());
|
||||
/// Compiled GREP styles by pattern; None for one that doesn't compile or gave up on a search.
|
||||
static PATTERNS: RefCell<HashMap<String, Option<Grep>>> = RefCell::new(HashMap::new());
|
||||
}
|
||||
|
||||
fn is_event(u: &NestedUntil, c: char, prev: Option<char>, next: Option<char>) -> bool {
|
||||
@@ -70,12 +73,15 @@ pub(crate) fn overlays(text: &str, range: Range<usize>, nested: &[NestedStyle],
|
||||
}
|
||||
PATTERNS.with(|p| {
|
||||
let mut p = p.borrow_mut();
|
||||
let re = p.entry(g.pattern.clone()).or_insert_with(|| regex::Regex::new(&g.pattern).ok());
|
||||
if let Some(re) = re {
|
||||
for m in re.find_iter(&text[range.start..end]) {
|
||||
if !m.is_empty() {
|
||||
out.push((range.start + m.start()..range.start + m.end(), g.style.clone()));
|
||||
}
|
||||
let slot = p.entry(g.pattern.clone()).or_insert_with(|| {
|
||||
Grep::new(&g.pattern, GrepOptions::default()).map_err(|e| log::warn!("GREP style {:?} skipped: {e}", g.pattern)).ok()
|
||||
});
|
||||
let Some(re) = slot else { return };
|
||||
match re.find_all(text.get(range.start..end).unwrap_or(""), usize::MAX) {
|
||||
Ok(ms) => out.extend(ms.into_iter().filter(|m| !m.is_empty()).map(|m| (range.start + m.start..range.start + m.end, g.style.clone()))),
|
||||
Err(e) => {
|
||||
log::warn!("GREP style {:?} disabled: {e}", g.pattern);
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -109,4 +115,13 @@ mod tests {
|
||||
// A bad pattern is ignored.
|
||||
assert!(overlays(&text, 0..text.len(), &[], &[GrepStyle { style: "X".into(), pattern: "(".into() }], 0).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grep_style_in_the_documents_dialect() {
|
||||
// Short Russian words kept with the next one: `\l` lowercase letter, `\h` horizontal space.
|
||||
let text = "Мы и он в доме.\n".to_string();
|
||||
let o = overlays(&text, 0..text.len(), &[], &[GrepStyle { style: "nobreak".into(), pattern: r"\b\l{1,2}\h".into() }], 0);
|
||||
let got: Vec<&str> = o.iter().map(|(r, _)| &text[r.clone()]).collect();
|
||||
assert_eq!(got, ["и ", "он ", "в "]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4668,3 +4668,31 @@ fn fixed_width_spaces_do_not_stretch_on_justified_lines() {
|
||||
}
|
||||
assert!(checked > 5, "{checked} fixed spaces on justified lines");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grep_style_in_the_documents_dialect_keeps_short_words_with_the_next() {
|
||||
// `\l` (lowercase letter) and `\h` (horizontal space) aren't Rust regex syntax; the style used
|
||||
// to be skipped silently.
|
||||
let text = "Мы и он в доме и в саду, а она у окна и с книгой в руках у стола.";
|
||||
let para = ParaAttrs {
|
||||
align: Some(Align::Left),
|
||||
grep_styles: Some(vec![designcraft_doc::GrepStyle { style: "nobreak".into(), pattern: r"\b\l{1,2}\h".into() }]),
|
||||
..Default::default()
|
||||
};
|
||||
let (mut d, sid, _) = doc_with(text, Rect::new(0.0, 0.0, 70.0, 1000.0), para);
|
||||
let chars = designcraft_doc::CharAttrs { no_break: Some(true), ..Default::default() };
|
||||
std::sync::Arc::make_mut(&mut d.styles).character.push(designcraft_doc::CharacterStyle {
|
||||
name: "nobreak".into(),
|
||||
based_on: None,
|
||||
chars,
|
||||
shortcut: String::new(),
|
||||
});
|
||||
let cs = compose_story(&d, sid, &ComposeOptions::default());
|
||||
let lines = all_lines(&cs);
|
||||
assert!(lines.len() > 3);
|
||||
let short = |w: &str| w.chars().count() <= 2 && w.chars().all(char::is_lowercase);
|
||||
for l in &lines[..lines.len() - 1] {
|
||||
let last = text[l.range.clone()].split_whitespace().last().unwrap_or("");
|
||||
assert!(!short(last), "line ends with a short word: {:?}", &text[l.range.clone()]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,6 @@ serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
quick-xml = { workspace = true }
|
||||
caseless = { workspace = true }
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Edit → Find/Change: plain text and GREP (regular expressions) with InDesign's metacharacters,
|
||||
//! optional formatting in the change, scoped to the document, a story or the selection.
|
||||
|
||||
use designcraft_compose::grep::{Grep, GrepOptions, translate_replacement};
|
||||
use designcraft_doc::{StoryId, TextSel, story};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -69,38 +70,19 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
]
|
||||
}
|
||||
|
||||
/// Translate InDesign text/GREP metacharacters to our story encoding.
|
||||
fn metachars(s: &str, grep: bool) -> String {
|
||||
let pairs: &[(&str, &str)] = if grep {
|
||||
&[
|
||||
("~b", "\n"),
|
||||
("\\r", "\n"),
|
||||
("~#", "\u{E000}"),
|
||||
("~x", "\u{E001}"),
|
||||
("~M", "\u{E002}"),
|
||||
("~R", "\u{E003}"),
|
||||
("~P", "\u{E004}"),
|
||||
("~L", "\u{E011}"),
|
||||
("~E", "\u{E012}"),
|
||||
("~=", "\u{2014}"),
|
||||
("~_", "\u{2014}"),
|
||||
("~-", "\u{AD}"),
|
||||
("~S", "\u{A0}"),
|
||||
("~n", "\u{2028}"),
|
||||
]
|
||||
} else {
|
||||
&[
|
||||
("^p", "\n"),
|
||||
("^t", "\t"),
|
||||
("^n", "\u{2028}"),
|
||||
("^#", "\u{E000}"),
|
||||
("^_", "\u{2014}"),
|
||||
("^=", "\u{2013}"),
|
||||
("^-", "\u{AD}"),
|
||||
("^s", "\u{A0}"),
|
||||
("^^", "^"),
|
||||
]
|
||||
};
|
||||
/// Translate text-search metacharacters (`^p`, `^t`…) to our story encoding.
|
||||
fn metachars(s: &str) -> String {
|
||||
let pairs: &[(&str, &str)] = &[
|
||||
("^p", "\n"),
|
||||
("^t", "\t"),
|
||||
("^n", "\u{2028}"),
|
||||
("^#", "\u{E000}"),
|
||||
("^_", "\u{2014}"),
|
||||
("^=", "\u{2013}"),
|
||||
("^-", "\u{AD}"),
|
||||
("^s", "\u{A0}"),
|
||||
("^^", "^"),
|
||||
];
|
||||
let mut out = s.to_string();
|
||||
for (a, b) in pairs {
|
||||
out = out.replace(a, b);
|
||||
@@ -108,21 +90,16 @@ fn metachars(s: &str, grep: bool) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
fn build(p: &Value) -> Result<regex::Regex> {
|
||||
/// The search pattern: GREP (in the documents' dialect, see `designcraft_compose::grep`) or
|
||||
/// literal text with metacharacters.
|
||||
fn build(p: &Value) -> Result<Grep> {
|
||||
let f = str_param(p, "find").ok_or_else(|| bad("find", "missing `find`"))?;
|
||||
if f.is_empty() {
|
||||
return Err(bad("find", "empty search"));
|
||||
}
|
||||
let grep = bool_or(p, "grep", false);
|
||||
let mut pat = if grep { metachars(f, true) } else { regex::escape(&metachars(f, false)) };
|
||||
if bool_or(p, "wholeWord", false) {
|
||||
pat = format!(r"\b(?:{pat})\b");
|
||||
}
|
||||
regex::RegexBuilder::new(&pat)
|
||||
.case_insensitive(!bool_or(p, "caseSensitive", false))
|
||||
.multi_line(true)
|
||||
.build()
|
||||
.map_err(|e| bad("find", format!("bad GREP: {e}")))
|
||||
let opts = GrepOptions { case_insensitive: !bool_or(p, "caseSensitive", false), multi_line: true, whole_word: bool_or(p, "wholeWord", false) };
|
||||
let re = if bool_or(p, "grep", false) { Grep::new(f, opts) } else { Grep::literal(&metachars(f), opts) };
|
||||
re.map_err(|e| bad("find", e))
|
||||
}
|
||||
|
||||
fn scope(s: &Session, p: &Value) -> Result<Vec<(StoryId, std::ops::Range<usize>)>> {
|
||||
@@ -146,27 +123,27 @@ fn scope(s: &Session, p: &Value) -> Result<Vec<(StoryId, std::ops::Range<usize>)
|
||||
}
|
||||
}
|
||||
|
||||
/// Most matches `find.find` returns.
|
||||
const MAX_MATCHES: usize = 10_000;
|
||||
|
||||
fn find(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let re = build(p)?;
|
||||
let st = s.doc()?;
|
||||
let mut out = Vec::new();
|
||||
for (sid, r) in scope(s, p)? {
|
||||
let Some(story) = st.doc.story(sid) else { continue };
|
||||
let text = &story.text[r.clone()];
|
||||
for m in re.find_iter(text) {
|
||||
out.push(json!({"story": sid.0, "start": r.start + m.start(), "end": r.start + m.end(), "text": m.as_str()}));
|
||||
if out.len() >= 10_000 {
|
||||
break;
|
||||
}
|
||||
let Some(text) = story.text.get(r.clone()) else { continue };
|
||||
for m in re.find_all(text, MAX_MATCHES.saturating_sub(out.len())).map_err(|e| bad("find", e))? {
|
||||
out.push(json!({"story": sid.0, "start": r.start + m.start, "end": r.start + m.end, "text": text.get(m)}));
|
||||
}
|
||||
}
|
||||
Ok(Value::Array(out))
|
||||
}
|
||||
|
||||
/// Expand `$1`… (GREP) or text metacharacters in the change string.
|
||||
/// The change string: an expansion template (`$1`…) for GREP, else literal text.
|
||||
fn replacement(p: &Value) -> String {
|
||||
let c = str_param(p, "change").unwrap_or("");
|
||||
metachars(c, bool_or(p, "grep", false))
|
||||
if bool_or(p, "grep", false) { translate_replacement(c) } else { metachars(c) }
|
||||
}
|
||||
|
||||
fn change(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
@@ -187,21 +164,16 @@ fn change(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
for (sid, r) in targets {
|
||||
let Some(story) = d.story(sid) else { continue };
|
||||
// Collect matches first (positions refer to the original text), apply back to front.
|
||||
let text = story.text[r.clone()].to_string();
|
||||
let mut edits: Vec<(std::ops::Range<usize>, String)> = Vec::new();
|
||||
for caps in re.captures_iter(&text) {
|
||||
let Some(m) = caps.get(0) else { continue };
|
||||
let mut out = String::new();
|
||||
if grep {
|
||||
caps.expand(&rep, &mut out);
|
||||
} else {
|
||||
out = rep.clone();
|
||||
}
|
||||
edits.push((r.start + m.start()..r.start + m.end(), out));
|
||||
if first_only {
|
||||
break;
|
||||
}
|
||||
let Some(text) = story.text.get(r.clone()) else { continue };
|
||||
let edits: Vec<(std::ops::Range<usize>, String)> = if grep {
|
||||
re.replacements(text, &rep, first_only)
|
||||
} else {
|
||||
re.find_all(text, if first_only { 1 } else { usize::MAX }).map(|ms| ms.into_iter().map(|m| (m, rep.clone())).collect())
|
||||
}
|
||||
.map_err(|e| bad("find.change", e))?
|
||||
.into_iter()
|
||||
.map(|(m, out)| (r.start + m.start..r.start + m.end, out))
|
||||
.collect();
|
||||
let Some(st) = d.story_mut(sid) else { continue };
|
||||
for (range, new) in edits.iter().rev() {
|
||||
st.replace(range.clone(), new);
|
||||
@@ -234,8 +206,8 @@ fn find_next(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
for (k, sid) in order.iter().enumerate() {
|
||||
let Some(story) = st.doc.story(*sid) else { continue };
|
||||
let from = if k == 0 && start_story == Some(*sid) { start_pos.min(story.len()) } else { 0 };
|
||||
if let Some(m) = re.find_at(&story.text, from) {
|
||||
hit = Some((*sid, m.start(), m.end()));
|
||||
if let Some(m) = re.find_at(&story.text, from).map_err(|e| bad("find.next", e))? {
|
||||
hit = Some((*sid, m.start, m.end));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -277,6 +249,26 @@ mod tests {
|
||||
s.execute("find.next", &json!({"find": "or"})).unwrap();
|
||||
assert!(s.doc().unwrap().selection.text.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grep_in_the_documents_dialect() {
|
||||
let mut s = Session::new();
|
||||
s.execute("file.new", &json!({})).unwrap();
|
||||
s.execute("frame.create", &json!({"rect": [36, 36, 300, 300], "content": "text", "text": "Москва and Paris, 1\u{2013}2 days.\nok"})).unwrap();
|
||||
// `\u` uppercase, `\l` lowercase letter.
|
||||
let m = s.execute("find.find", &json!({"find": r"\u\l+", "grep": true, "caseSensitive": true})).unwrap();
|
||||
let words: Vec<&str> = m.as_array().unwrap().iter().map(|x| x["text"].as_str().unwrap()).collect();
|
||||
assert_eq!(words, ["Москва", "Paris"]);
|
||||
// `~=` en dash, `\h` space, `\r` paragraph end; the change uses groups, `~S` and `\r`.
|
||||
let r = s.execute("find.change", &json!({"find": r"(\d)~=(\d)\h(\l+)\.\r", "change": r"$1~=$2~S$3!\r", "grep": true})).unwrap();
|
||||
assert_eq!(r["count"], 1);
|
||||
let sid = s.execute("document.inspect", &json!({})).unwrap()["stories"][0]["id"].as_u64().unwrap();
|
||||
let text = s.execute("story.get", &json!({"story": sid})).unwrap()["text"].as_str().unwrap().to_string();
|
||||
assert!(text.starts_with("Москва and Paris, 1\u{2013}2\u{A0}days!\nok"), "{text:?}");
|
||||
// A pattern that doesn't compile is an error, not a silent no-op.
|
||||
assert!(s.execute("find.find", &json!({"find": "[[:nope:]]", "grep": true})).is_err());
|
||||
assert!(s.execute("find.change", &json!({"find": "(", "change": "", "grep": true})).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// Objects matching Find/Change › Object criteria, in document order (nested ones included).
|
||||
|
||||
@@ -20,7 +20,6 @@ designcraft-engine = { workspace = true }
|
||||
unicode-bidi = { workspace = true }
|
||||
unicode-bidi-mirroring = { workspace = true }
|
||||
unicode-segmentation = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
egui = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -2239,7 +2239,7 @@ fn regex_ok(p: &str) -> Result<(), ()> {
|
||||
if p.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
regex::Regex::new(p).map(|_| ()).map_err(|_| ())
|
||||
designcraft_compose::grep::Grep::new(p, Default::default()).map(|_| ()).map_err(|_| ())
|
||||
}
|
||||
|
||||
fn paragraph_style_options(app: &mut DesignApp, ui: &mut egui::Ui, d: &mut Dialog) {
|
||||
|
||||
Reference in New Issue
Block a user