aleleba/vectorcraft
1
0
Fork 0
mirror of https://github.com/storytold/vectorcraft.git synced 2026-11-11 02:19:56 -06:00
Code Issues Packages Projects Releases Wiki Activity

JavaScript: text built-ins hold their working memory against the heap limit

Browse Source 
Native code now holds the Rust memory it works in against the heap limit
until it is done with it: a matcher's capture state and stack, a pattern
being compiled, the matches and pieces of a replacement, and the text
Date.parse reads. A collection counts these bytes as live while they are
held.

- replace reads its replacement pattern once into parts that refer to
  ranges of the pattern, so literal text is not copied for each match.
  Each piece takes fuel for its units, and the result's length is
  checked against the heap as each piece is added.
- The regular expression compiler charges heap through its callback as
  it builds, including the ranges of a character class, and each class
  atom and range takes fuel. A compiled form counts the capacity of its
  lists.
- Clearing the groups inside a quantifier and clearing the matcher's
  state before each attempt take fuel.
- Flattening a rope in a String, RegExp or Date method reserves the
  flat copy on the heap and takes fuel for it.
- A RegExp object's bytes are reserved together with its compiled form,
  so a collection between the two does not drop the compiled form from
  the heap count.
This commit is contained in:
dotCooCoo
2026-11-07 19:29:53 -08:00
parent 02fffcd00c
commit cea958b782
6 changed files with 582 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/js/src/date_builtins.rs
+10 -5
View File
@@ -13,8 +13,8 @@ use chrono::{Datelike, NaiveDate};
use crate::builtins::{arg, constructor, data, method, native};
use crate::heap::{Class, NativeFn, NativeKind, Object, key};
use crate::interp::{ErrorType, Hint, Interp, JsResult};
use crate::string_builtins::{charge_units, reserve_units, to_integer};
use crate::interp::{ErrorType, HeldBytes, Hint, Interp, JsResult};
use crate::string_builtins::{charge_units, flat_charged, reserve_units, to_integer};
use crate::value::{Str, Value};
/// The clock and time zone a realm's `Date` reads.
@@ -425,10 +425,15 @@ fn parse_text(interp: &Interp, s: &str) -> Option<f64> {
Some(time_clip(t))
}
/// Takes fuel for the units of `s` the parsers read.
/// Takes fuel for the units of `s` the parsers read. The text the parsers read is held against
/// the heap limit while they read it.
fn parse_date(interp: &mut Interp, s: &Str) -> JsResult<f64> {
charge_units(interp, s.len())?;
let text = s.to_rust_string();
let units = flat_charged(interp, s)?;
charge_units(interp, units.len())?;
let mut held = HeldBytes::new(interp);
// Each code unit becomes at most three UTF-8 bytes.
held.add(interp, units.len().saturating_mul(3))?;
let text = String::from_utf16_lossy(&units);
let trimmed = text.trim();
Ok(parse_iso(trimmed.as_bytes()).or_else(|| parse_text(interp, trimmed)).unwrap_or(f64::NAN))
}
crates/js/src/interp.rs
+48 -2
View File
@@ -166,6 +166,50 @@ pub(crate) struct Interp {
pub(crate) last_thrown: Value,
/// The clock and time zone `Date` reads.
pub(crate) date_host: crate::date_builtins::DateHost,
/// The bytes native code holds outside the heap's cells through [`HeldBytes`].
pub(crate) held: Rc<HeldCount>,
}
/// The bytes all live [`HeldBytes`] hold.
#[derive(Default)]
pub(crate) struct HeldCount {
now: std::cell::Cell<usize>,
}
/// Bytes that native code holds in Rust memory while it works (a matcher's state and stack, a
/// replacement being built, a pattern being compiled), counted against the heap limit. A
/// collection adds them to the live total until the holder is dropped.
pub(crate) struct HeldBytes {
count: Rc<HeldCount>,
bytes: usize,
}
impl HeldBytes {
pub(crate) fn new(interp: &Interp) -> Self {
Self { count: interp.held.clone(), bytes: 0 }
}
/// The bytes this holder holds.
#[cfg(test)]
pub(crate) fn bytes(&self) -> usize {
self.bytes
}
/// Holds `n` more bytes, ending the run with the heap limit when they would not fit after a
/// collection. Everything the caller still needs must be reachable from the roots.
pub(crate) fn add(&mut self, interp: &mut Interp, n: usize) -> JsResult<()> {
interp.reserve_heap(n)?;
interp.heap.bytes = interp.heap.bytes.saturating_add(n);
self.bytes = self.bytes.saturating_add(n);
self.count.now.set(self.count.now.get().saturating_add(n));
Ok(())
}
}
impl Drop for HeldBytes {
fn drop(&mut self) {
self.count.now.set(self.count.now.get().saturating_sub(self.bytes));
}
}
/// The least heap size a collection waits for.
@@ -292,6 +336,7 @@ impl Interp {
throw_site: None,
last_thrown: Value::Undefined,
date_host: crate::date_builtins::DateHost::default(),
held: Rc::default(),
};
crate::builtins::install(&mut interp);
interp.gc_threshold = interp.next_threshold();
@@ -418,8 +463,8 @@ impl Interp {
Ok(())
}
/// Collects garbage now, for a caller whose work ran out of heap room, and sets when the
/// next collection runs.
/// Collects garbage now and sets when the next collection runs.
#[cfg(test)]
pub(crate) fn collect_for_room(&mut self) {
self.collect_garbage();
self.gc_threshold = self.next_threshold();
@@ -452,6 +497,7 @@ impl Interp {
roots.objects.extend(self.intrinsics.all());
roots.envs.push(self.global_env);
self.heap.collect(&roots);
self.heap.bytes = self.heap.bytes.saturating_add(self.held.now.get());
}
/// Runs frames until the boundary frame on top returns, and returns its value.
crates/js/src/regexp_builtins.rs
+29 -30
View File
@@ -3,18 +3,17 @@
use std::rc::Rc;
use crate::LimitKind;
use crate::builtins::{arg, constructor, method, native};
use crate::heap::{Class, NativeKind, OBJECT_BYTES, OPEN, ObjId, Object, PROP_BYTES, Prop, WRITABLE, key, string_bytes};
use crate::interp::{Abrupt, ErrorType, Interp, JsResult};
use crate::regexp_vm::{CompileFail, Flags, Matcher, Regex, compile, parse_flags, use_fuel_n};
use crate::string_builtins::{new_array, reserve_units, rooted, set_element, substring_value, to_integer};
use crate::interp::{ErrorType, HeldBytes, Interp, JsResult};
use crate::regexp_vm::{CompileFail, Cost, Flags, Matcher, Regex, compile, parse_flags, use_fuel_n};
use crate::string_builtins::{flat_charged, new_array, reserve_units, rooted, set_element, substring_value, to_integer};
use crate::value::{Str, Value};
pub(crate) fn install(interp: &mut Interp) -> JsResult<()> {
let proto = interp.intrinsics.regexp_proto;
// The prototype is itself a regular expression object, matching the empty string.
let empty = compile_regex(interp, &[], &[])?;
let empty = compile_regex(interp, &[], &[], 0)?;
if let Some(p) = interp.heap.object_mut(proto) {
p.class = Class::RegExp(empty);
}
@@ -36,8 +35,10 @@ pub(crate) fn regexp_of(interp: &Interp, v: &Value) -> Option<(ObjId, Rc<Regex>)
}
/// Checks and compiles a pattern, throwing a `SyntaxError` for a pattern or flags that are not
/// valid. The compiled form's bytes are reserved on the heap.
fn compile_regex(interp: &mut Interp, pattern: &[u16], flags: &[u16]) -> JsResult<Rc<Regex>> {
/// valid. What compiling holds is held against the heap limit as it grows. The compiled form's
/// bytes are then reserved together with `extra` more and added to the heap, so a caller that
/// stores the result in an object it allocates next reserves that object here.
fn compile_regex(interp: &mut Interp, pattern: &[u16], flags: &[u16], extra: usize) -> JsResult<Rc<Regex>> {
let flags: Flags = match parse_flags(flags) {
Ok(f) => f,
Err(m) => return Err(interp.throw(ErrorType::Syntax, &m)),
@@ -46,30 +47,26 @@ fn compile_regex(interp: &mut Interp, pattern: &[u16], flags: &[u16]) -> JsResul
if let Err(m) = crate::regexp::check(pattern, depth) {
return Err(interp.throw(ErrorType::Syntax, &format!("invalid regular expression: {m}")));
}
// A compile that runs out of room collects garbage and tries once more with the room it frees.
let mut collected = false;
let re = loop {
let room = interp.limits.heap_bytes.saturating_sub(interp.heap.bytes);
match compile(pattern, flags, depth, room, &mut |n| use_fuel_n(interp, n)) {
Ok(re) => break re,
Err(CompileFail::Abrupt(Abrupt::Limit(LimitKind::Heap))) if !collected => {
interp.collect_for_room();
collected = true;
}
Err(CompileFail::Syntax(m)) => return Err(interp.throw(ErrorType::Syntax, &format!("invalid regular expression: {m}"))),
Err(CompileFail::Abrupt(a)) => return Err(a),
}
let mut held = HeldBytes::new(interp);
let compiled = compile(pattern, flags, depth, &mut |cost| match cost {
Cost::Fuel(n) => use_fuel_n(interp, n),
Cost::Heap(n) => held.add(interp, n),
});
drop(held);
let re = match compiled {
Ok(re) => re,
Err(CompileFail::Syntax(m)) => return Err(interp.throw(ErrorType::Syntax, &format!("invalid regular expression: {m}"))),
Err(CompileFail::Abrupt(a)) => return Err(a),
};
let bytes = re.heap_bytes();
interp.reserve_heap(bytes)?;
interp.reserve_heap(bytes.saturating_add(extra))?;
interp.heap.bytes = interp.heap.bytes.saturating_add(bytes);
Ok(Rc::new(re))
}
/// A new RegExp object whose `source` property is `source` (section 15.10.4.1, steps 7 to 12).
fn make(interp: &mut Interp, source: Str, pattern: &[u16], flags: &[u16]) -> JsResult<ObjId> {
let re = compile_regex(interp, pattern, flags)?;
interp.reserve_heap(OBJECT_BYTES + 5 * (PROP_BYTES + string_bytes(10)))?;
let re = compile_regex(interp, pattern, flags, OBJECT_BYTES + 5 * (PROP_BYTES + string_bytes(10)))?;
let (g, i, m) = (parse_flags(flags).unwrap_or_default().global, re.ignore_case, re.multiline);
let proto = interp.intrinsics.regexp_proto;
let r = interp.heap.alloc_object(Object::new(Class::RegExp(re), Some(proto)));
@@ -91,8 +88,9 @@ fn define_flags(interp: &mut Interp, r: ObjId, source: Str, g: bool, i: bool, m:
/// The object a regular expression literal evaluates to (section 7.8.5): a new RegExp object
/// each time.
pub(crate) fn new_regexp(interp: &mut Interp, source: &Str, flags: &Str) -> JsResult<ObjId> {
let pattern = source.flat();
make(interp, source.clone(), &pattern, &flags.flat())
let pattern = flat_charged(interp, source)?;
let flags = flat_charged(interp, flags)?;
make(interp, source.clone(), &pattern, &flags)
}
/// The `source` text for a pattern given to the constructor: `(?:)` for the empty pattern, and
@@ -153,15 +151,16 @@ pub(crate) fn new_regexp_from(interp: &mut Interp, pattern: &Value, flags: &Valu
f.push(c);
}
}
let units = source.flat();
let units = flat_charged(interp, &source)?;
let pattern: &[u16] = if is_empty_group(&units) { &[] } else { &units };
return make(interp, source.clone(), pattern, &f.encode_utf16().collect::<Vec<u16>>());
}
let p = if matches!(pattern, Value::Undefined) { Str::from("") } else { interp.to_str(pattern)? };
let f = if matches!(flags, Value::Undefined) { Str::from("") } else { interp.to_str(flags)? };
let units = p.flat();
let units = flat_charged(interp, &p)?;
let flags = flat_charged(interp, &f)?;
let source = source_text(interp, &units)?;
make(interp, source, &units, &f.flat())
make(interp, source, &units, &flags)
}
fn is_empty_group(units: &[u16]) -> bool {
@@ -194,7 +193,7 @@ pub(crate) fn exec_raw(interp: &mut Interp, rx: ObjId, s: &Str) -> JsResult<Opti
let li = interp.get_value(&rv, &last_index)?;
let li = to_integer(interp.to_number(&li)?);
let global = interp.get_value(&rv, &key("global"))?.truthy();
let units = s.flat();
let units = flat_charged(interp, s)?;
let i = if global { li } else { 0.0 };
let found = if i < 0.0 || i > units.len() as f64 {
None
@@ -221,7 +220,7 @@ pub(crate) fn exec_raw(interp: &mut Interp, rx: ObjId, s: &Str) -> JsResult<Opti
pub(crate) fn exec_value(interp: &mut Interp, rx: ObjId, s: &Str) -> JsResult<Value> {
rooted(interp, |interp| {
let Some(caps) = exec_raw(interp, rx, s)? else { return Ok(Value::Null) };
let units = s.flat();
let units = flat_charged(interp, s)?;
let a = new_array(interp);
interp.reserve_heap(2 * (PROP_BYTES + string_bytes(5)))?;
let index = caps.first().copied().unwrap_or(0);
crates/js/src/regexp_vm.rs
+185 -90
View File
@@ -5,14 +5,15 @@
//! pattern with a loop and a stack of open groups, and the matcher keeps its choice points and the
//! values it must restore on its own stack, so neither recurses however deeply the groups nest.
//! Quantifiers count their iterations in registers instead of repeating their atom's code. Every
//! instruction run and every choice point taken back costs fuel, and the matcher's stack is
//! charged against the heap limit, so a pattern that backtracks without end stops at a limit.
//! instruction run, every capture slot cleared and every choice point taken back costs fuel, and
//! the compiler's work, the matcher's state and its stack are held against the heap limit, so a
//! pattern that backtracks without end stops at a limit.
use std::collections::HashMap;
use std::sync::OnceLock;
use crate::LimitKind;
use crate::interp::{Abrupt, Interp, JsResult};
use crate::interp::{Abrupt, HeldBytes, Interp, JsResult};
/// A capture position that is not set.
pub(crate) const NONE: usize = usize::MAX;
@@ -131,10 +132,21 @@ impl From<&str> for CompileFail {
}
}
/// What compiling takes as it goes: fuel for the work done, or heap bytes for what it holds.
#[derive(Clone, Copy, Debug)]
pub(crate) enum Cost {
Fuel(u64),
Heap(usize),
}
impl Regex {
/// The estimated bytes the compiled form holds.
/// The bytes the compiled form holds, counting the capacity of each list.
pub(crate) fn heap_bytes(&self) -> usize {
64 + size_of::<Inst>() * self.insts.len() + self.sets.iter().map(set_bytes).sum::<usize>()
let sets = self.sets.iter().map(set_bytes).fold(0usize, usize::saturating_add);
size_of::<Regex>()
.saturating_add(size_of::<Inst>().saturating_mul(self.insts.capacity()))
.saturating_add(size_of::<CharSet>().saturating_mul(self.sets.capacity()))
.saturating_add(sets)
}
fn canon(&self, u: u16) -> u16 {
@@ -239,6 +251,7 @@ fn complement(ranges: &[(u16, u16)]) -> Vec<(u16, u16)> {
if next <= 0xFFFF {
out.push((next as u16, 0xFFFF));
}
out.shrink_to_fit();
out
}
@@ -252,6 +265,7 @@ fn normalize(mut ranges: Vec<(u16, u16)>) -> Vec<(u16, u16)> {
_ => out.push((lo, hi)),
}
}
out.shrink_to_fit();
out
}
@@ -313,9 +327,10 @@ impl Frags {
Item::Frag(self.items.len() - 1)
}
/// The instructions of `top` with every fragment in place, written with a loop.
/// The instructions of `top` with every fragment in place, written with a loop, with room for
/// one more.
fn flatten(&self, top: &[Item]) -> Vec<Inst> {
let mut out = Vec::with_capacity(self.len_of(top));
let mut out = Vec::with_capacity(self.len_of(top).saturating_add(1));
let mut stack: Vec<(&[Item], usize)> = vec![(top, 0)];
while let Some((items, i)) = stack.pop() {
let Some(item) = items.get(i) else { continue };
@@ -493,22 +508,64 @@ fn class_atom(pattern: &[u16], i: usize) -> Result<(ClassAtom, usize), CompileFa
}
}
/// The character class whose `[` is at `i`, and the position after its `]`.
fn class(pattern: &[u16], mut i: usize, room: usize) -> Result<(CharSet, usize), CompileFail> {
/// Counts `n` units of work in `work`, taking them from the fuel once a batch is full.
fn add_work(work: &mut u64, n: u64, charge: &mut dyn FnMut(Cost) -> JsResult<()>) -> Result<(), CompileFail> {
*work = work.saturating_add(n);
if *work >= FUEL_BATCH {
charge(Cost::Fuel(std::mem::take(work))).map_err(CompileFail::Abrupt)?;
}
Ok(())
}
/// The ranges of a class being read, with the bytes charged for them so far.
struct ClassRanges {
ranges: Vec<(u16, u16)>,
held: usize,
}
impl ClassRanges {
/// Adds the ranges of an atom.
fn add_atom(&mut self, atom: &ClassAtom, work: &mut u64, charge: &mut dyn FnMut(Cost) -> JsResult<()>) -> Result<(), CompileFail> {
match atom {
ClassAtom::Unit(u) => self.add(&[(*u, *u)], work, charge),
ClassAtom::Ranges(r) => self.add(r, work, charge),
}
}
/// Adds `part`. Each range counts as work, and when the list must grow, the bytes of its new
/// capacity and of the copy `normalize` makes of it are charged first.
fn add(&mut self, part: &[(u16, u16)], work: &mut u64, charge: &mut dyn FnMut(Cost) -> JsResult<()>) -> Result<(), CompileFail> {
let extra = part.len();
add_work(work, extra as u64, charge)?;
let len = self.ranges.len().saturating_add(extra);
if len > self.ranges.capacity() {
let cap = len.max(self.ranges.capacity().saturating_mul(2));
let need = cap.saturating_mul(2 * size_of::<(u16, u16)>());
if need > self.held {
charge(Cost::Heap(need - self.held)).map_err(CompileFail::Abrupt)?;
self.held = need;
}
self.ranges.reserve_exact(cap - self.ranges.len());
}
self.ranges.extend_from_slice(part);
Ok(())
}
}
/// The character class whose `[` is at `i`, and the position after its `]`. Each atom read and
/// each range added counts as work in `work`, and the class's ranges are charged as they grow.
fn class(pattern: &[u16], mut i: usize, work: &mut u64, charge: &mut dyn FnMut(Cost) -> JsResult<()>) -> Result<(CharSet, usize), CompileFail> {
i += 1;
let negated = pattern.get(i) == Some(&0x5E);
if negated {
i += 1;
}
let mut ranges = Vec::new();
let mut list = ClassRanges { ranges: Vec::new(), held: 0 };
loop {
// The ranges and the copy `normalize` makes of them stay within `room`.
if ranges.len().saturating_mul(2 * size_of::<(u16, u16)>()) > room {
return Err(CompileFail::Abrupt(Abrupt::Limit(LimitKind::Heap)));
}
add_work(work, 1, charge)?;
let u = *pattern.get(i).ok_or("unterminated character class")?;
if u == 0x5D {
return Ok((CharSet { ranges: normalize(ranges), negated }, i + 1));
return Ok((CharSet { ranges: normalize(list.ranges), negated }, i + 1));
}
let (atom, next) = class_atom(pattern, i)?;
i = next;
@@ -520,31 +577,28 @@ fn class(pattern: &[u16], mut i: usize, room: usize) -> Result<(CharSet, usize),
if lo > hi {
return Err("range out of order in character class".into());
}
ranges.push((lo, hi));
list.add(&[(lo, hi)], work, charge)?;
}
// Annex B: a range with a class escape at one end is its atoms and a `-`.
(lo, hi) => {
for a in [lo, ClassAtom::Unit(0x2D), hi] {
match a {
ClassAtom::Unit(u) => ranges.push((u, u)),
ClassAtom::Ranges(r) => ranges.extend(r),
}
list.add_atom(&a, work, charge)?;
}
}
}
i = after;
} else {
match atom {
ClassAtom::Unit(u) => ranges.push((u, u)),
ClassAtom::Ranges(r) => ranges.extend(r),
}
list.add_atom(&atom, work, charge)?;
}
}
}
/// The bytes a character class holds.
/// At least the bytes a class escape's set (`\d`, `\S`, ...) holds.
const ESCAPE_SET_BYTES: usize = size_of::<CharSet>() + 16 * size_of::<(u16, u16)>();
/// The bytes a character class's ranges hold.
fn set_bytes(set: &CharSet) -> usize {
32 + size_of::<(u16, u16)>() * set.ranges.len()
size_of::<(u16, u16)>().saturating_mul(set.ranges.capacity())
}
/// The bytes compiling holds: each instruction as an item and in the flattened code, each
@@ -556,16 +610,11 @@ pub(crate) fn compile_bytes(instructions: usize, fragments: usize, sets_size: us
.saturating_add(sets_size)
}
/// Compiles `pattern`. Groups may nest `max_depth` deep; `charge` takes fuel for the work done.
/// Compiling stops with the heap limit when what it holds (instructions, fragments and classes
/// made so far, with the two instructions each `|` adds) would pass `room` bytes.
pub(crate) fn compile(
pattern: &[u16],
flags: Flags,
max_depth: usize,
room: usize,
charge: &mut dyn FnMut(u64) -> JsResult<()>,
) -> Result<Regex, CompileFail> {
/// Compiles `pattern`. Groups may nest `max_depth` deep. `charge` takes fuel for the work done, and
/// heap bytes for what compiling holds before it grows: the instructions, fragments and classes
/// made so far (with the two instructions each `|` adds), the groups still open and the ranges of
/// the class being read. An error from `charge` stops compiling.
pub(crate) fn compile(pattern: &[u16], flags: Flags, max_depth: usize, charge: &mut dyn FnMut(Cost) -> JsResult<()>) -> Result<Regex, CompileFail> {
let total_groups = count_groups(pattern);
let mut sets: Vec<CharSet> = Vec::new();
let mut sets_size = 0usize;
@@ -579,15 +628,19 @@ pub(crate) fn compile(
let mut emitted = 0usize;
// The fragments the joins of the groups still open will add, one for each `|` read in them.
let mut joined = 0usize;
// The heap bytes charged so far.
let mut held = 0usize;
let canon = |u: u16| if flags.ignore_case { canonicalize(u) } else { u };
while let Some(&u) = pattern.get(i) {
work += 1;
if work >= FUEL_BATCH {
charge(work).map_err(CompileFail::Abrupt)?;
work = 0;
}
if compile_bytes(emitted, frags.items.len() + joined, sets_size) > room {
return Err(CompileFail::Abrupt(Abrupt::Limit(LimitKind::Heap)));
add_work(&mut work, 1, charge)?;
// One step adds at most three instructions, two fragments, one open group and one class
// escape's set (a class charges its own ranges), so their bytes are charged before the
// step makes them.
let now = compile_bytes(emitted + 3, frags.items.len() + joined + 2, sets_size.saturating_add(ESCAPE_SET_BYTES))
.saturating_add(size_of::<Ctx>().saturating_mul(stack.len() + 1));
if now > held {
charge(Cost::Heap(now - held)).map_err(CompileFail::Abrupt)?;
held = now;
}
let ctx = stack.last_mut().ok_or("unmatched ')'")?;
let start = ctx.cur.len();
@@ -612,9 +665,9 @@ pub(crate) fn compile(
i += 1;
}
0x5B => {
let (set, next) = class(pattern, i, room.saturating_sub(compile_bytes(emitted, frags.items.len() + joined, sets_size)))?;
let (set, next) = class(pattern, i, &mut work, charge)?;
emit(&mut emitted, ctx, Inst::Set(u32::try_from(sets.len()).map_err(|_| "too many classes")?));
sets_size = sets_size.saturating_add(set_bytes(&set));
sets_size = sets_size.saturating_add(size_of::<CharSet>() + set_bytes(&set));
sets.push(set);
ctx.atom = Some((start, before));
i = next;
@@ -746,7 +799,7 @@ pub(crate) fn compile(
if let Some(r) = escape_ranges(next) {
emit(&mut emitted, ctx, Inst::Set(u32::try_from(sets.len()).map_err(|_| "too many classes")?));
let set = CharSet { ranges: r, negated: false };
sets_size = sets_size.saturating_add(set_bytes(&set));
sets_size = sets_size.saturating_add(size_of::<CharSet>() + set_bytes(&set));
sets.push(set);
i += 2;
} else {
@@ -775,13 +828,15 @@ pub(crate) fn compile(
let cur = std::mem::take(&mut top.cur);
top.alts.push(cur);
// Joining adds a fragment for each alternative, and flattening writes every instruction.
if compile_bytes(emitted + 1, frags.items.len() + joined + 1, sets_size) > room {
return Err(CompileFail::Abrupt(Abrupt::Limit(LimitKind::Heap)));
let now = compile_bytes(emitted + 1, frags.items.len() + joined + 1, sets_size);
if now > held {
charge(Cost::Heap(now - held)).map_err(CompileFail::Abrupt)?;
}
let top = join(top.alts, &mut frags)?;
let mut insts = frags.flatten(&top);
insts.push(Inst::Match);
charge(work + insts.len() as u64 / 16).map_err(CompileFail::Abrupt)?;
sets.shrink_to_fit();
charge(Cost::Fuel(work + insts.len() as u64 / 16)).map_err(CompileFail::Abrupt)?;
Ok(Regex { insts, sets, groups: groups as usize, loops: loops as usize, ignore_case: flags.ignore_case, multiline: flags.multiline })
}
@@ -830,7 +885,12 @@ pub(crate) fn use_fuel_n(interp: &mut Interp, n: u64) -> JsResult<()> {
interp.use_fuel()
}
/// The state of one match attempt, kept between attempts to reuse its buffers.
/// The stack entries a matcher's stack first grows by.
const STACK_CHUNK: usize = 64;
/// The state of one match attempt, kept between attempts to reuse its buffers. Its capture and
/// quantifier state, and its stack's capacity, are held against the heap limit until it is
/// dropped.
pub(crate) struct Matcher<'r> {
re: &'r Regex,
/// Two positions per group, group 0 first: [`NONE`] where a group did not take part.
@@ -839,25 +899,20 @@ pub(crate) struct Matcher<'r> {
counts: Vec<u64>,
marks: Vec<usize>,
stack: Vec<Bt>,
max_stack: usize,
/// The stack entries held.
stack_room: usize,
held: HeldBytes,
steps: u64,
}
impl<'r> Matcher<'r> {
/// A matcher for `re`. Its capture and quantifier state, and its stack, fit in the heap room
/// left under the limit; a run without room for the state ends with the heap limit. When the
/// room would not hold the state and 64 stack entries, a collection runs first, so everything
/// the caller still needs must be reachable from the roots.
/// A matcher for `re`, ending the run with the heap limit when its state does not fit. Holding
/// the state, and later the stack, can run a collection, so everything the caller still needs
/// must be reachable from the roots while the matcher works.
pub(crate) fn new(re: &'r Regex, interp: &mut Interp) -> JsResult<Self> {
let state = 8usize.saturating_mul(re.groups.saturating_add(1).saturating_mul(3).saturating_add(re.loops.saturating_mul(2)));
let wanted = state.saturating_add(64 * size_of::<Bt>());
if interp.heap.bytes.saturating_add(wanted) > interp.limits.heap_bytes {
interp.collect_for_room();
}
let room = interp.limits.heap_bytes.saturating_sub(interp.heap.bytes);
if state > room {
return Err(Abrupt::Limit(LimitKind::Heap));
}
let mut held = HeldBytes::new(interp);
held.add(interp, state)?;
Ok(Self {
re,
caps: vec![NONE; 2 * (re.groups + 1)],
@@ -865,19 +920,43 @@ impl<'r> Matcher<'r> {
counts: vec![0; re.loops],
marks: vec![NONE; re.loops],
stack: Vec::new(),
max_stack: (room - state) / size_of::<Bt>(),
stack_room: 0,
held,
steps: 0,
})
}
fn push(&mut self, b: Bt) -> JsResult<()> {
if self.stack.len() >= self.max_stack {
return Err(Abrupt::Limit(LimitKind::Heap));
/// The bytes the matcher holds.
#[cfg(test)]
pub(crate) fn held_bytes(&self) -> usize {
self.held.bytes()
}
fn push(&mut self, interp: &mut Interp, b: Bt) -> JsResult<()> {
if self.stack.len() >= self.stack_room {
self.grow(interp)?;
}
self.stack.push(b);
Ok(())
}
/// Holds more stack entries: as many as the stack has, at least [`STACK_CHUNK`], or when those
/// do not fit, what the heap room left holds.
fn grow(&mut self, interp: &mut Interp) -> JsResult<()> {
let size = size_of::<Bt>();
let mut more = self.stack_room.max(STACK_CHUNK);
if self.held.add(interp, more.saturating_mul(size)).is_err() {
more = interp.limits.heap_bytes.saturating_sub(interp.heap.bytes) / size;
if more == 0 {
return Err(Abrupt::Limit(LimitKind::Heap));
}
self.held.add(interp, more * size)?;
}
self.stack_room = self.stack_room.saturating_add(more);
self.stack.reserve_exact(self.stack_room - self.stack.len());
Ok(())
}
fn step(&mut self, interp: &mut Interp) -> JsResult<()> {
self.steps += 1;
if self.steps >= FUEL_BATCH {
@@ -893,10 +972,10 @@ impl<'r> Matcher<'r> {
use_fuel_n(interp, n)
}
fn set_cap(&mut self, slot: usize, v: usize) -> JsResult<()> {
fn set_cap(&mut self, interp: &mut Interp, slot: usize, v: usize) -> JsResult<()> {
let Some(old) = self.caps.get(slot).copied() else { return Ok(()) };
if old != v {
self.push(Bt::Cap(slot, old))?;
self.push(interp, Bt::Cap(slot, old))?;
if let Some(c) = self.caps.get_mut(slot) {
*c = v;
}
@@ -904,17 +983,20 @@ impl<'r> Matcher<'r> {
Ok(())
}
fn enter_rep(&mut self, pc: usize, pos: usize) -> JsResult<()> {
/// Starts an iteration of the quantifier at `pc`, clearing the groups inside its atom. Each
/// group cleared counts as a step.
fn enter_rep(&mut self, interp: &mut Interp, pc: usize, pos: usize) -> JsResult<()> {
let Some(Inst::Rep { reg, caps: (lo, hi), .. }) = self.re.insts.get(pc).copied() else { return Ok(()) };
let reg = reg as usize;
let old = self.marks.get(reg).copied().unwrap_or(NONE);
self.push(Bt::Mark(reg, old))?;
self.push(interp, Bt::Mark(reg, old))?;
if let Some(m) = self.marks.get_mut(reg) {
*m = pos;
}
for g in lo as usize..hi as usize {
self.set_cap(2 * g, NONE)?;
self.set_cap(2 * g + 1, NONE)?;
self.step(interp)?;
self.set_cap(interp, 2 * g, NONE)?;
self.set_cap(interp, 2 * g + 1, NONE)?;
}
Ok(())
}
@@ -950,7 +1032,7 @@ impl<'r> Matcher<'r> {
}
Bt::EnterRep { pc: p, pos: q } => {
*pos = q;
self.enter_rep(p, q)?;
self.enter_rep(interp, p, q)?;
*pc = p + 1;
return Ok(true);
}
@@ -985,7 +1067,10 @@ impl<'r> Matcher<'r> {
Ok(false)
}
/// Clears the state for a new attempt. Each slot cleared counts as a step.
fn reset(&mut self) {
let slots = self.caps.len() + self.open.len() + self.counts.len() + self.marks.len();
self.steps = self.steps.saturating_add(slots as u64);
self.caps.fill(NONE);
self.open.fill(NONE);
self.counts.fill(0);
@@ -1047,7 +1132,7 @@ impl<'r> Matcher<'r> {
(a != b) == want
}
Inst::Split(first, second) => {
self.push(Bt::Retry { pc: at(pc, second), pos })?;
self.push(interp, Bt::Retry { pc: at(pc, second), pos })?;
pc = at(pc, first);
true
}
@@ -1058,7 +1143,7 @@ impl<'r> Matcher<'r> {
Inst::GroupStart(g) => {
let g = g as usize;
let old = self.open.get(g).copied().unwrap_or(NONE);
self.push(Bt::Open(g, old))?;
self.push(interp, Bt::Open(g, old))?;
if let Some(o) = self.open.get_mut(g) {
*o = pos;
}
@@ -1068,15 +1153,15 @@ impl<'r> Matcher<'r> {
Inst::GroupEnd(g) => {
let g = g as usize;
let s = self.open.get(g).copied().unwrap_or(NONE);
self.set_cap(2 * g, s)?;
self.set_cap(2 * g + 1, pos)?;
self.set_cap(interp, 2 * g, s)?;
self.set_cap(interp, 2 * g + 1, pos)?;
pc += 1;
true
}
Inst::RepInit(r) => {
let r = r as usize;
let old = self.counts.get(r).copied().unwrap_or(0);
self.push(Bt::Count(r, old))?;
self.push(interp, Bt::Count(r, old))?;
if let Some(c) = self.counts.get_mut(r) {
*c = 0;
}
@@ -1089,14 +1174,14 @@ impl<'r> Matcher<'r> {
if count >= max {
pc = exit_pc;
} else if count < min {
self.enter_rep(pc, pos)?;
self.enter_rep(interp, pc, pos)?;
pc += 1;
} else if greedy {
self.push(Bt::Retry { pc: exit_pc, pos })?;
self.enter_rep(pc, pos)?;
self.push(interp, Bt::Retry { pc: exit_pc, pos })?;
self.enter_rep(interp, pc, pos)?;
pc += 1;
} else {
self.push(Bt::EnterRep { pc, pos })?;
self.push(interp, Bt::EnterRep { pc, pos })?;
pc = exit_pc;
}
true
@@ -1114,7 +1199,7 @@ impl<'r> Matcher<'r> {
if count >= min && self.marks.get(r).copied() == Some(pos) {
false
} else {
self.push(Bt::Count(r, count))?;
self.push(interp, Bt::Count(r, count))?;
if let Some(c) = self.counts.get_mut(r) {
*c = count + 1;
}
@@ -1123,7 +1208,7 @@ impl<'r> Matcher<'r> {
}
}
Inst::Look { negative, end } => {
self.push(Bt::Look { pos, negative, resume: at(pc, end) })?;
self.push(interp, Bt::Look { pos, negative, resume: at(pc, end) })?;
pc += 1;
true
}
@@ -1131,6 +1216,8 @@ impl<'r> Matcher<'r> {
let Some(b) = self.stack.iter().rposition(|e| matches!(e, Bt::Look { .. })) else {
return Err(Abrupt::Internal("lookahead end without its start".into()));
};
// Each entry read above, and below, counts as a step.
self.steps = self.steps.saturating_add((self.stack.len() - b) as u64);
let Some(Bt::Look { pos: q, negative, resume }) = self.stack.get(b).copied() else {
return Err(Abrupt::Internal("lookahead end without its start".into()));
};
@@ -1146,9 +1233,17 @@ impl<'r> Matcher<'r> {
false
} else {
// A lookahead is atomic: its choice points are dropped, and what it set is
// kept, with the entries that restore it.
let kept: Vec<Bt> = self.stack.drain(b..).skip(1).filter(Bt::is_undo).collect();
self.stack.extend(kept);
// kept, with the entries that restore it, moved down in place.
let mut kept = b;
for r in b + 1..self.stack.len() {
if let Some(e) = self.stack.get(r).copied().filter(Bt::is_undo) {
if let Some(slot) = self.stack.get_mut(kept) {
*slot = e;
}
kept += 1;
}
}
self.stack.truncate(kept);
pos = q;
pc = resume;
true
@@ -1175,8 +1270,8 @@ impl<'r> Matcher<'r> {
}
}
Inst::Match => {
self.set_cap(0, start)?;
self.set_cap(1, pos)?;
self.set_cap(interp, 0, start)?;
self.set_cap(interp, 1, pos)?;
return Ok(true);
}
};
crates/js/src/string_builtins.rs
+192 -103
View File
@@ -1,17 +1,17 @@
//! `String` (ECMA-262 5.1 section 15.5): the constructor, `String.fromCharCode` and the methods of
//! `String.prototype`, with `substr` from Annex B.
//!
//! The methods read a string's code units once (flattening a rope), take a unit of fuel for every
//! [`UNITS_PER_FUEL`] code units they read or write, and reserve each new string's bytes on the
//! heap before building it, so a method that would make a string past the heap limit ends the
//! run instead of allocating it.
//! The methods read a string's code units once (flattening a rope, whose flat copy counts as a new
//! string), take a unit of fuel for every [`UNITS_PER_FUEL`] code units they read or write, and
//! reserve each new string's bytes on the heap before building it, so a method that would make a
//! string past the heap limit ends the run instead of allocating it.
use std::rc::Rc;
use crate::LimitKind;
use crate::builtins::{arg, constructor, method, native};
use crate::heap::{Class, NativeKind, OPEN, ObjId, Object, PROP_BYTES, Prop, string_bytes};
use crate::interp::{Abrupt, Interp, JsResult};
use crate::interp::{Abrupt, HeldBytes, Interp, JsResult};
use crate::regexp_builtins::{exec_raw, new_regexp_from, regexp_of};
use crate::regexp_vm::{Matcher, NONE, use_fuel_n};
use crate::value::{Str, Value, index_key, is_str_space, to_uint32};
@@ -74,13 +74,47 @@ pub(crate) fn charge_units(interp: &mut Interp, n: usize) -> JsResult<()> {
/// run with the heap limit when it would not fit. Everything the caller still needs must be
/// reachable from the roots.
pub(crate) fn reserve_units(interp: &mut Interp, len: usize) -> JsResult<()> {
reserve_string(interp, len)?;
charge_units(interp, len)
}
/// [`reserve_units`] without the fuel, for a caller that took it as it read the units.
fn reserve_string(interp: &mut Interp, len: usize) -> JsResult<()> {
check_string_room(interp, len)?;
let bytes = string_bytes(len);
interp.heap.bytes = interp.heap.bytes.saturating_add(bytes);
Ok(())
}
/// Ends the run with the heap limit when a new string of `len` code units would not fit, after
/// a collection if one is needed. Everything the caller still needs must be reachable from the
/// roots.
fn check_string_room(interp: &mut Interp, len: usize) -> JsResult<()> {
if len > interp.limits.heap_bytes / 4 {
return Err(Abrupt::Limit(LimitKind::Heap));
}
let bytes = string_bytes(len);
interp.reserve_heap(bytes)?;
interp.heap.bytes = interp.heap.bytes.saturating_add(bytes);
charge_units(interp, len)
interp.reserve_heap(string_bytes(len))
}
/// Pushes `item` onto `list`, first holding the bytes of the list's new capacity when it must
/// grow.
fn push_held<T>(interp: &mut Interp, held: &mut HeldBytes, list: &mut Vec<T>, item: T) -> JsResult<()> {
if list.len() == list.capacity() {
let more = list.capacity().max(4);
held.add(interp, more.saturating_mul(size_of::<T>()))?;
list.reserve_exact(more);
}
list.push(item);
Ok(())
}
/// The code units of `s`. Flattening a rope makes a copy of its units, so for a rope that copy's
/// bytes are reserved and added to the heap, and its units charged as fuel, first.
pub(crate) fn flat_charged(interp: &mut Interp, s: &Str) -> JsResult<Rc<[u16]>> {
if s.is_rope() {
reserve_units(interp, s.len())?;
}
Ok(s.flat())
}
/// A new string holding `units[start..end]`.
@@ -215,7 +249,7 @@ fn char_at(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResul
if pos < 0.0 || pos >= s.len() as f64 {
return Ok(Value::str(""));
}
let unit = s.flat().get(pos as usize).copied().unwrap_or(0);
let unit = flat_charged(interp, &s)?.get(pos as usize).copied().unwrap_or(0);
Ok(Value::String(Str::from_units(vec![unit])))
}
@@ -225,7 +259,7 @@ fn char_code_at(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> Js
if pos < 0.0 || pos >= s.len() as f64 {
return Ok(Value::Number(f64::NAN));
}
Ok(Value::Number(f64::from(s.flat().get(pos as usize).copied().unwrap_or(0))))
Ok(Value::Number(f64::from(flat_charged(interp, &s)?.get(pos as usize).copied().unwrap_or(0))))
}
fn concat(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<Value> {
@@ -241,7 +275,7 @@ fn index_of(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResu
let s = this_str(interp, &this, "indexOf")?;
let search = interp.to_str(&arg(args, 0))?;
let pos = int_arg(interp, args, 1)?;
let (hay, needle) = (s.flat(), search.flat());
let (hay, needle) = (flat_charged(interp, &s)?, flat_charged(interp, &search)?);
let start = clamp(pos, hay.len());
let found = find(interp, &hay, &needle, start)?;
Ok(Value::Number(found.map_or(-1.0, |i| i as f64)))
@@ -252,7 +286,7 @@ fn last_index_of(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> J
let search = interp.to_str(&arg(args, 0))?;
let n = interp.to_number(&arg(args, 1))?;
let pos = if n.is_nan() { f64::INFINITY } else { to_integer(n) };
let (hay, needle) = (s.flat(), search.flat());
let (hay, needle) = (flat_charged(interp, &s)?, flat_charged(interp, &search)?);
let start = clamp(pos, hay.len());
if needle.len() > hay.len() {
return Ok(Value::Number(-1.0));
@@ -275,7 +309,7 @@ fn last_index_of(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> J
fn locale_compare(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<Value> {
let s = this_str(interp, &this, "localeCompare")?;
let that = interp.to_str(&arg(args, 0))?;
let (a, b) = (s.flat(), that.flat());
let (a, b) = (flat_charged(interp, &s)?, flat_charged(interp, &that)?);
charge_units(interp, a.len().min(b.len()))?;
Ok(Value::Number(match a.cmp(&b) {
std::cmp::Ordering::Less => -1.0,
@@ -294,7 +328,8 @@ fn slice(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<
if to <= from {
return Ok(Value::str(""));
}
substring_value(interp, &s.flat(), from as usize, to as usize)
let units = flat_charged(interp, &s)?;
substring_value(interp, &units, from as usize, to as usize)
}
fn substring(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<Value> {
@@ -303,7 +338,8 @@ fn substring(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsRes
let start = int_arg(interp, args, 0)?;
let end = if matches!(arg(args, 1), Value::Undefined) { len as f64 } else { int_arg(interp, args, 1)? };
let (a, b) = (clamp(start, len), clamp(end, len));
substring_value(interp, &s.flat(), a.min(b), a.max(b))
let units = flat_charged(interp, &s)?;
substring_value(interp, &units, a.min(b), a.max(b))
}
/// `substr` (Annex B.2.3). Its first step applies ToString to `this` with no object-coercible
@@ -318,7 +354,8 @@ fn substr(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult
if count <= 0.0 {
return Ok(Value::str(""));
}
substring_value(interp, &s.flat(), start as usize, (start + count) as usize)
let units = flat_charged(interp, &s)?;
substring_value(interp, &units, start as usize, (start + count) as usize)
}
/// The UTF-16 length of `c` mapped to lower or upper case.
@@ -353,19 +390,21 @@ fn map_case(interp: &mut Interp, units: &[u16], upper: bool) -> JsResult<Vec<u16
fn to_lower_case(interp: &mut Interp, this: Value, _: &[Value], _: bool) -> JsResult<Value> {
let s = this_str(interp, &this, "toLowerCase")?;
let out = map_case(interp, &s.flat(), false)?;
let units = flat_charged(interp, &s)?;
let out = map_case(interp, &units, false)?;
Ok(Value::String(Str::from_units(out)))
}
fn to_upper_case(interp: &mut Interp, this: Value, _: &[Value], _: bool) -> JsResult<Value> {
let s = this_str(interp, &this, "toUpperCase")?;
let out = map_case(interp, &s.flat(), true)?;
let units = flat_charged(interp, &s)?;
let out = map_case(interp, &units, true)?;
Ok(Value::String(Str::from_units(out)))
}
fn trim(interp: &mut Interp, this: Value, _: &[Value], _: bool) -> JsResult<Value> {
let s = this_str(interp, &this, "trim")?;
let units = s.flat();
let units = flat_charged(interp, &s)?;
// The scan reads every unit once at most.
charge_units(interp, units.len())?;
let start = units.iter().position(|&u| !is_str_space(u)).unwrap_or(units.len());
@@ -403,7 +442,7 @@ fn match_(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult
let a = new_array(interp);
let mut n = 0usize;
let mut previous = 0.0;
let units = s.flat();
let units = flat_charged(interp, &s)?;
while let Some(caps) = exec_raw(interp, rx, &s)? {
let this_index = interp.get_value(&Value::Object(rx), &last_index)?;
let this_index = interp.to_number(&this_index)?;
@@ -428,54 +467,106 @@ fn search(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult
let s = this_str(interp, &this, "search")?;
let rx = regexp_arg(interp, &arg(args, 0))?;
let Some((_, re)) = regexp_of(interp, &Value::Object(rx)) else { return Err(interp.type_error("not a regular expression")) };
let units = s.flat();
let units = flat_charged(interp, &s)?;
let mut m = Matcher::new(&re, interp)?;
let found = m.search(interp, &units, 0)?;
Ok(Value::Number(if found { m.caps.first().copied().unwrap_or(0) as f64 } else { -1.0 }))
})
}
/// A piece of a result string: a range of the input, or other text.
/// A piece of a result string: a range of the input, a range of the replacement pattern, or the
/// units of a string a replacement function returned.
enum Piece {
Range(usize, usize),
Text(Str),
Template(usize, usize),
Units(Rc<[u16]>),
}
impl Piece {
fn len(&self) -> usize {
match self {
Piece::Range(a, b) => b.saturating_sub(*a),
Piece::Text(s) => s.len(),
Piece::Range(a, b) | Piece::Template(a, b) => b.saturating_sub(*a),
Piece::Units(u) => u.len(),
}
}
}
/// Joins pieces into a new string, reserving its length first.
fn build(interp: &mut Interp, units: &[u16], pieces: &[Piece]) -> JsResult<Value> {
let total = pieces.iter().map(Piece::len).fold(0usize, usize::saturating_add);
reserve_units(interp, total)?;
let mut out = Vec::with_capacity(total);
for p in pieces {
match p {
Piece::Range(a, b) => out.extend_from_slice(units.get(*a..*b).unwrap_or_default()),
Piece::Text(s) => out.extend_from_slice(&s.flat()),
}
}
Ok(Value::String(Str::from_units(out)))
/// The pieces of a result string and the length they add up to.
#[derive(Default)]
struct Pieces {
list: Vec<Piece>,
len: usize,
}
/// The pieces a replacement pattern (section 15.5.4.11, Table 22) gives for one match: `caps`
/// holds the match and its groups as pairs of positions in `units`.
fn expand(template: &[u16], units: &[u16], caps: &[usize], out: &mut Vec<Piece>) {
let groups = caps.len() / 2 - 1;
let (start, end) = (caps.first().copied().unwrap_or(0), caps.get(1).copied().unwrap_or(0));
impl Pieces {
/// Adds `piece`, holding its bytes and taking fuel for its units, and ends the run with the
/// heap limit once the result would not fit.
fn add(&mut self, interp: &mut Interp, held: &mut HeldBytes, piece: Piece) -> JsResult<()> {
let n = piece.len();
if n == 0 {
return Ok(());
}
push_held(interp, held, &mut self.list, piece)?;
self.len = self.len.saturating_add(n);
charge_units(interp, n)?;
check_string_room(interp, self.len)
}
/// Joins the pieces into a new string, reserving its length first. `units` is the input and
/// `template` the replacement pattern.
fn build(&self, interp: &mut Interp, units: &[u16], template: &[u16]) -> JsResult<Value> {
reserve_string(interp, self.len)?;
let mut out = Vec::with_capacity(self.len);
for p in &self.list {
match p {
Piece::Range(a, b) => out.extend_from_slice(units.get(*a..*b).unwrap_or_default()),
Piece::Template(a, b) => out.extend_from_slice(template.get(*a..*b).unwrap_or_default()),
Piece::Units(u) => out.extend_from_slice(u),
}
}
Ok(Value::String(Str::from_units(out)))
}
}
/// A part of a replacement pattern (section 15.5.4.11, Table 22).
#[derive(Clone, Copy)]
enum Part {
/// The pattern's own text in a range.
Text(usize, usize),
/// `$&`
Matched,
/// `` $` ``
Before,
/// `$'`
After,
/// `$n` or `$nn`
Group(usize),
}
impl Part {
/// The piece this part gives for one match: `caps` holds the match and its groups as pairs of
/// positions in an input of `len` units.
fn piece(self, caps: &[usize], len: usize) -> Option<Piece> {
let (start, end) = (caps.first().copied().unwrap_or(0), caps.get(1).copied().unwrap_or(0));
match self {
Part::Text(a, b) => Some(Piece::Template(a, b)),
Part::Matched => Some(Piece::Range(start, end)),
Part::Before => Some(Piece::Range(0, start)),
Part::After => Some(Piece::Range(end, len)),
Part::Group(n) => {
let (s, e) = (caps.get(2 * n).copied().unwrap_or(NONE), caps.get(2 * n + 1).copied().unwrap_or(NONE));
(s != NONE && e != NONE).then_some(Piece::Range(s, e))
}
}
}
}
/// Reads a replacement pattern once into its parts, for a regular expression with `groups`
/// groups. The parts are held against the heap limit, and the units read take fuel.
fn parse_template(interp: &mut Interp, held: &mut HeldBytes, template: &[u16], groups: usize) -> JsResult<Vec<Part>> {
charge_units(interp, template.len())?;
let digit = |i: usize| template.get(i).filter(|&&u| (0x30..=0x39).contains(&u)).map(|&u| usize::from(u - 0x30));
let group = |n: usize, out: &mut Vec<Piece>| {
let (s, e) = (caps.get(2 * n).copied().unwrap_or(NONE), caps.get(2 * n + 1).copied().unwrap_or(NONE));
if s != NONE && e != NONE {
out.push(Piece::Range(s, e));
}
};
let mut parts = Vec::new();
let mut lit = 0;
let mut i = 0;
while let Some(&u) = template.get(i) {
@@ -483,65 +574,49 @@ fn expand(template: &[u16], units: &[u16], caps: &[usize], out: &mut Vec<Piece>)
i += 1;
continue;
}
let flush = |out: &mut Vec<Piece>, to: usize| {
if to > lit {
out.push(Piece::Text(Str::from_units(template.get(lit..to).unwrap_or_default().to_vec())));
}
};
let taken = match template.get(i + 1) {
let (part, taken) = match template.get(i + 1) {
// `$$` gives one `$`: the text up to and including the first.
Some(0x24) => {
flush(out, i + 1);
push_held(interp, held, &mut parts, Part::Text(lit, i + 1))?;
lit = i + 2;
i += 2;
continue;
}
Some(0x26) => {
flush(out, i);
out.push(Piece::Range(start, end));
2
}
Some(0x60) => {
flush(out, i);
out.push(Piece::Range(0, start));
2
}
Some(0x27) => {
flush(out, i);
out.push(Piece::Range(end, units.len()));
2
}
Some(0x26) => (Part::Matched, 2),
Some(0x60) => (Part::Before, 2),
Some(0x27) => (Part::After, 2),
_ => match (digit(i + 1), digit(i + 2)) {
(Some(a), Some(b)) if (1..=groups).contains(&(a * 10 + b)) => {
flush(out, i);
group(a * 10 + b, out);
3
}
(Some(a), _) if (1..=groups).contains(&a) => {
flush(out, i);
group(a, out);
2
}
(Some(a), Some(b)) if (1..=groups).contains(&(a * 10 + b)) => (Part::Group(a * 10 + b), 3),
(Some(a), _) if (1..=groups).contains(&a) => (Part::Group(a), 2),
_ => {
i += 1;
continue;
}
},
};
if i > lit {
push_held(interp, held, &mut parts, Part::Text(lit, i))?;
}
push_held(interp, held, &mut parts, part)?;
i += taken;
lit = i;
}
if template.len() > lit {
out.push(Piece::Text(Str::from_units(template.get(lit..).unwrap_or_default().to_vec())));
push_held(interp, held, &mut parts, Part::Text(lit, template.len()))?;
}
Ok(parts)
}
/// `replace` (section 15.5.4.11).
/// `replace` (section 15.5.4.11). The matches, the parsed replacement pattern and the pieces of
/// the result are held against the heap limit while the result is put together, and the
/// result's length is checked against the heap as each piece is added.
fn replace(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<Value> {
rooted(interp, |interp| {
let s = this_str(interp, &this, "replace")?;
let search_value = arg(args, 0);
let replace_value = arg(args, 1);
let units = s.flat();
let units = flat_charged(interp, &s)?;
let mut held = HeldBytes::new(interp);
// Each match as the positions of the match and its groups.
let mut matches: Vec<Vec<usize>> = Vec::new();
if let Some((rx, _)) = regexp_of(interp, &search_value) {
@@ -550,12 +625,9 @@ fn replace(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResul
let last_index = crate::heap::key("lastIndex");
interp.put(rx, last_index.clone(), Value::Number(0.0), true)?;
let mut previous = 0.0;
// The bytes the stored matches and the pieces of the result made from them hold,
// kept within the heap room left.
let mut stored = 0usize;
while let Some(caps) = exec_raw(interp, rx, &s)? {
stored = stored.saturating_add(24 + size_of::<usize>() * caps.len() + 2 * size_of::<Piece>());
interp.reserve_heap(stored)?;
held.add(interp, size_of::<usize>().saturating_mul(caps.capacity()))?;
push_held(interp, &mut held, &mut matches, caps)?;
let this_index = interp.get_value(&search_value, &last_index)?;
let this_index = interp.to_number(&this_index)?;
if this_index == previous {
@@ -564,14 +636,13 @@ fn replace(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResul
} else {
previous = this_index;
}
matches.push(caps);
}
} else if let Some(caps) = exec_raw(interp, rx, &s)? {
matches.push(caps);
}
} else {
let search = interp.to_str(&search_value)?;
let needle = search.flat();
let needle = flat_charged(interp, &search)?;
if let Some(i) = find(interp, &units, &needle, 0)? {
matches.push(vec![i, i + needle.len()]);
}
@@ -579,18 +650,25 @@ fn replace(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResul
let replacer = interp.is_callable(&replace_value).then(|| replace_value.clone());
let template = match &replacer {
Some(_) => None,
None => Some(interp.to_str(&replace_value)?.flat()),
None => {
let t = interp.to_str(&replace_value)?;
Some(flat_charged(interp, &t)?)
}
};
if matches.is_empty() {
return Ok(Value::String(s));
}
let mut pieces = Vec::new();
let template: &[u16] = template.as_deref().unwrap_or_default();
// Every match has the same number of groups.
let groups = matches.first().map_or(0, |caps| (caps.len() / 2).saturating_sub(1));
let parts = if replacer.is_none() { parse_template(interp, &mut held, template, groups)? } else { Vec::new() };
let mut pieces = Pieces::default();
let mut last = 0usize;
for caps in &matches {
let (start, end) = (caps.first().copied().unwrap_or(0), caps.get(1).copied().unwrap_or(0));
pieces.push(Piece::Range(last, start));
match (&replacer, &template) {
(Some(f), _) => {
pieces.add(interp, &mut held, Piece::Range(last, start))?;
match &replacer {
Some(f) => {
let mut call_args = Vec::with_capacity(caps.len() / 2 + 2);
for pair in caps.chunks(2) {
let v = match pair {
@@ -603,16 +681,24 @@ fn replace(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResul
call_args.push(Value::String(s.clone()));
let r = interp.call(f, &Value::Undefined, &call_args)?;
let text = interp.to_str(&r)?;
pieces.push(Piece::Text(text));
let text = flat_charged(interp, &text)?;
pieces.add(interp, &mut held, Piece::Units(text))?;
}
None => {
// Reading the parts takes fuel even when they give empty pieces.
use_fuel_n(interp, (parts.len() / UNITS_PER_FUEL) as u64)?;
for part in &parts {
if let Some(p) = part.piece(caps, units.len()) {
pieces.add(interp, &mut held, p)?;
}
}
}
(None, Some(t)) => expand(t, &units, caps, &mut pieces),
(None, None) => {}
}
use_fuel_n(interp, 1)?;
last = end;
}
pieces.push(Piece::Range(last, units.len()));
build(interp, &units, &pieces)
pieces.add(interp, &mut held, Piece::Range(last, units.len()))?;
pieces.build(interp, &units, template)
})
}
@@ -627,7 +713,10 @@ fn split(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<
let rx = regexp_of(interp, &separator);
let sep = match &rx {
Some(_) => None,
None => Some(interp.to_str(&separator)?.flat()),
None => {
let sep = interp.to_str(&separator)?;
Some(flat_charged(interp, &sep)?)
}
};
if lim == 0 {
return Ok(Value::Object(a));
@@ -636,7 +725,7 @@ fn split(interp: &mut Interp, this: Value, args: &[Value], _: bool) -> JsResult<
set_element(interp, a, 0, Value::String(s))?;
return Ok(Value::Object(a));
}
let units = s.flat();
let units = flat_charged(interp, &s)?;
let size = units.len();
let re: Option<Rc<crate::regexp_vm::Regex>> = rx.map(|(_, re)| re);
let mut matcher = re.as_deref().map(|re| Matcher::new(re, interp)).transpose()?;
crates/js/src/text_tests.rs
+118 -22
View File
@@ -226,7 +226,7 @@ fn a_catastrophic_regular_expression_ends_by_fuel() {
#[test]
fn trim_and_date_parsing_take_fuel_for_the_units_they_scan() {
let limits = Limits { fuel: 40_000, ..Limits::default() };
let build = "var s = ' '; for (var i = 0; i < 20; i++) s = s + s; s.charAt(0)";
let build = "var s = ' '; for (var i = 0; i < 20; i++) s = s + s; s.length";
assert!(Realm::with_options(Options::default(), limits.clone()).run(build).is_ok());
for scan in ["s.trim()", "Date.parse(s)", "new Date(s)"] {
let e = fail_with(Realm::with_options(Options::default(), limits.clone()), &format!("{build}; {scan}"));
@@ -242,6 +242,58 @@ fn global_replace_counts_its_matches_against_the_heap() {
assert_eq!(run(src), JsValue::Number(0.0));
}
#[test]
fn replace_holds_its_pieces_and_checks_the_result_as_it_grows() {
// Each `$&` gives a piece for every match: the pieces count against the heap, so the run ends
// with the heap limit long before the 4,194,304 pieces would take the fuel.
let limits = Limits { heap_bytes: 4 << 20, fuel: 500_000, ..Limits::default() };
let src = "var s = 'a'; for (var i = 0; i < 12; i++) s += s; var t = '$&'; for (var i = 0; i < 10; i++) t += t; s.replace(/a/g, t).length";
let e = fail_with(Realm::with_options(Options::default(), limits), src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Heap), "{e:?}");
// A long literal pattern is not copied for each match, and the result's length is checked as
// the pieces are added, so the run ends with the heap limit before the fuel for 16,384
// copies of the pattern is spent.
let limits = Limits { heap_bytes: 4 << 20, fuel: 1_000_000, ..Limits::default() };
let src = "var s = 'a'; for (var i = 0; i < 14; i++) s += s; var t = 'x'; for (var i = 0; i < 11; i++) t += t; s.replace(/a/g, t).length";
let e = fail_with(Realm::with_options(Options::default(), limits), src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Heap), "{e:?}");
// The parts of the pattern give the same result as before.
shows("'abcabc'.replace(/(b)(c)/g, '[$$|$&|$`|$\\'|$2$1|$3|$0|$10]')", "a[$|bc|a|abc|cb|$3|$0|b0]a[$|bc|abca||cb|$3|$0|b0]");
}
#[test]
fn flattening_a_rope_counts_against_the_heap() {
// Three ropes of about 524,288 units, each 1 MiB once flat, with a 3 MiB heap.
let build = "var s = 'x'; for (var i = 0; i < 19; i++) s = s + s; var t = s + 'y', u = s + 'z';";
let limits = Limits { heap_bytes: 3 << 20, ..Limits::default() };
let built = Realm::with_options(Options::default(), limits.clone()).run(&format!("{build} s.length + t.length + u.length"));
assert_eq!(built, Ok(JsValue::Number(1_572_866.0)));
for read in [
"charAt(0)",
"charCodeAt(0)",
"indexOf('q')",
"lastIndexOf('q')",
"localeCompare('a')",
"slice(0, 1)",
"substring(0, 1)",
"substr(0, 1)",
"trim()",
"match(/q/)",
"search(/q/)",
"replace(/q/, '')",
"replace('q', '')",
] {
let src = format!("{build} s.{read}; t.{read}; u.{read}; 0");
let e = fail_with(Realm::with_options(Options::default(), limits.clone()), &src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Heap), "{read}: {e:?}");
}
for call in ["/q/.exec(X)", "/q/.test(X)", "Date.parse(X)", "new RegExp(X)"] {
let src = format!("{build} {}; {}; {}; 0", call.replace('X', "s"), call.replace('X', "t"), call.replace('X', "u"));
let e = fail_with(Realm::with_options(Options::default(), limits.clone()), &src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Heap), "{call}: {e:?}");
}
}
#[test]
fn compiling_a_pattern_counts_against_the_heap() {
let src = "var s = '[\\\\s\\\\S]a*'; for (var i = 0; i < 14; i++) s = s + s; new RegExp(s).source.length";
@@ -249,25 +301,47 @@ fn compiling_a_pattern_counts_against_the_heap() {
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Heap), "{e:?}");
assert_eq!(run(src), JsValue::Number(131_072.0));
// The compiler stops while it reads the pattern, before the code it builds passes the room.
let pattern: Vec<u16> = "[\\s\\S]a*".repeat(1000).encode_utf16().collect();
let flags = crate::regexp_vm::parse_flags(&[]).unwrap();
let room = crate::regexp_vm::compile_bytes(400, 0, 0);
let result = crate::regexp_vm::compile(&pattern, flags, 200, room, &mut |_| Ok(()));
assert!(matches!(result, Err(crate::regexp_vm::CompileFail::Abrupt(crate::interp::Abrupt::Limit(LimitKind::Heap)))));
assert!(crate::regexp_vm::compile(&pattern, flags, 200, usize::MAX, &mut |_| Ok(())).is_ok());
let pattern = "[\\s\\S]a*".repeat(1000);
assert!(stopped_by(&compile_within(&pattern, crate::regexp_vm::compile_bytes(400, 0, 0), u64::MAX).0, LimitKind::Heap));
assert!(compile_within(&pattern, usize::MAX, u64::MAX).0.is_ok());
// A literal pattern takes what its instructions hold, not the most a unit can take.
let literal: Vec<u16> = "a".repeat(10_000).encode_utf16().collect();
let room = crate::regexp_vm::compile_bytes(20_000, 0, 0);
assert!(crate::regexp_vm::compile(&literal, flags, 200, room, &mut |_| Ok(())).is_ok());
assert!(compile_within(&"a".repeat(10_000), crate::regexp_vm::compile_bytes(20_000, 0, 0), u64::MAX).0.is_ok());
// A class's ranges count while they are read, even when they collapse into one range.
let class: Vec<u16> = format!("[{}]", "a".repeat(10_000)).encode_utf16().collect();
let result = crate::regexp_vm::compile(&class, flags, 200, 1000, &mut |_| Ok(()));
assert!(matches!(result, Err(crate::regexp_vm::CompileFail::Abrupt(crate::interp::Abrupt::Limit(LimitKind::Heap)))));
assert!(stopped_by(&compile_within(&format!("[{}]", "a".repeat(10_000)), 1000, u64::MAX).0, LimitKind::Heap));
// So do the ranges of class escapes, and each atom and range read takes fuel.
let escapes = format!("[{}]", "\\w".repeat(10_000));
assert!(stopped_by(&compile_within(&escapes, 64 << 10, u64::MAX).0, LimitKind::Heap));
assert!(stopped_by(&compile_within(&escapes, usize::MAX, 20_000).0, LimitKind::Fuel));
// The joining and flattening after the last unit count too.
let ab: Vec<u16> = "ab".encode_utf16().collect();
let room = crate::regexp_vm::compile_bytes(1, 0, 0);
let result = crate::regexp_vm::compile(&ab, flags, 200, room, &mut |_| Ok(()));
assert!(matches!(result, Err(crate::regexp_vm::CompileFail::Abrupt(crate::interp::Abrupt::Limit(LimitKind::Heap)))));
assert!(stopped_by(&compile_within("ab", crate::regexp_vm::compile_bytes(1, 0, 0), u64::MAX).0, LimitKind::Heap));
}
/// Compiles `pattern` with no flags through a charge that allows `heap` bytes and `fuel` units,
/// and returns the result with the bytes and fuel the compiler asked for.
fn compile_within(pattern: &str, heap: usize, fuel: u64) -> (Result<crate::regexp_vm::Regex, crate::regexp_vm::CompileFail>, usize, u64) {
use crate::regexp_vm::Cost;
let units: Vec<u16> = pattern.encode_utf16().collect();
let flags = crate::regexp_vm::parse_flags(&[]).unwrap();
let (mut bytes, mut used) = (0usize, 0u64);
let result = crate::regexp_vm::compile(&units, flags, 200, &mut |cost| {
match cost {
Cost::Heap(n) => bytes = bytes.saturating_add(n),
Cost::Fuel(n) => used = used.saturating_add(n),
}
if bytes > heap {
return Err(crate::interp::Abrupt::Limit(LimitKind::Heap));
}
if used > fuel {
return Err(crate::interp::Abrupt::Limit(LimitKind::Fuel));
}
Ok(())
});
(result, bytes, used)
}
/// Whether compiling stopped with the limit `kind`.
fn stopped_by(result: &Result<crate::regexp_vm::Regex, crate::regexp_vm::CompileFail>, kind: LimitKind) -> bool {
matches!(result, Err(crate::regexp_vm::CompileFail::Abrupt(crate::interp::Abrupt::Limit(k))) if *k == kind)
}
#[test]
@@ -295,9 +369,7 @@ fn a_constructed_source_counts_against_the_heap_before_it_is_built() {
#[test]
fn the_matcher_state_counts_against_the_heap() {
let mut realm = Realm::with_options(Options::default(), Limits { heap_bytes: 1 << 20, ..Limits::default() });
let pattern: Vec<u16> = "(a)".repeat(1000).encode_utf16().collect();
let flags = crate::regexp_vm::parse_flags(&[]).unwrap();
let Ok(re) = crate::regexp_vm::compile(&pattern, flags, 200, usize::MAX, &mut |_| Ok(())) else { panic!("the pattern compiles") };
let Ok(re) = compile_within(&"(a)".repeat(1000), usize::MAX, u64::MAX).0 else { panic!("the pattern compiles") };
let interp = realm.interp();
interp.collect_for_room();
let live = interp.heap.bytes;
@@ -307,15 +379,39 @@ fn the_matcher_state_counts_against_the_heap() {
assert!(crate::regexp_vm::Matcher::new(&re, interp).is_ok());
// The stack takes only the room left: with room for the state and no more, backtracking ends
// with the heap limit.
let pattern: Vec<u16> = "a*b".encode_utf16().collect();
let Ok(re) = crate::regexp_vm::compile(&pattern, flags, 200, usize::MAX, &mut |_| Ok(())) else { panic!("the pattern compiles") };
let Ok(re) = compile_within("a*b", usize::MAX, u64::MAX).0 else { panic!("the pattern compiles") };
let input: Vec<u16> = "aaaaaaaaaa".encode_utf16().collect();
interp.limits.heap_bytes = live + 64;
let Ok(mut m) = crate::regexp_vm::Matcher::new(&re, interp) else { panic!("the state fits") };
assert!(matches!(m.search(interp, &input, 0), Err(crate::interp::Abrupt::Limit(LimitKind::Heap))));
drop(m);
interp.limits.heap_bytes = 1 << 20;
let Ok(mut m) = crate::regexp_vm::Matcher::new(&re, interp) else { panic!("the state fits") };
assert!(matches!(m.search(interp, &input, 0), Ok(false)));
// What the matcher holds is on the heap while it lives, so other allocations cannot use the
// same room, and a collection after it is dropped takes it off again.
let held = m.held_bytes();
assert!(held > 0);
assert!(interp.heap.bytes >= live + held, "{} < {live} + {held}", interp.heap.bytes);
interp.collect_for_room();
assert!(interp.heap.bytes >= live + held, "{} < {live} + {held}", interp.heap.bytes);
drop(m);
interp.collect_for_room();
assert_eq!(interp.heap.bytes, live);
}
#[test]
fn clearing_matcher_state_takes_fuel() {
let limits = Limits { fuel: 500_000, ..Limits::default() };
let groups = "var p = ''; for (var i = 0; i < 1000; i++) p += '(x)'; var s = ''; for (var i = 0; i < 2000; i++) s += 'a';";
// Each iteration of the `*` clears the 1000 groups inside it, though only `a` runs.
let src = format!("{groups} new RegExp('(?:a|' + p + ')*').test(s)");
let e = fail_with(Realm::with_options(Options::default(), limits.clone()), &src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Fuel), "{e:?}");
// Each attempt at a new position clears the state of all 1001 groups.
let src = format!("{groups} new RegExp(p + 'z').test(s.replace(/a/g, 'b'))");
let e = fail_with(Realm::with_options(Options::default(), limits), &src);
assert_eq!(e.kind, RunErrorKind::Limit(LimitKind::Fuel), "{e:?}");
}
#[test]