diff --git a/Cargo.lock b/Cargo.lock index d06f43371..620d90904 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6264,6 +6264,7 @@ dependencies = [ "vectorcraft-eps", "vectorcraft-format", "vectorcraft-geom", + "vectorcraft-js", "vectorcraft-metafile", "vectorcraft-pathops", "vectorcraft-pdf", @@ -6332,6 +6333,14 @@ dependencies = [ "vectorcraft-testkit", ] +[[package]] +name = "vectorcraft-js" +version = "0.21.0" +dependencies = [ + "proptest", + "unicode-general-category", +] + [[package]] name = "vectorcraft-mcp" version = "0.21.0" diff --git a/Cargo.toml b/Cargo.toml index 1f81e7569..a3e3e9890 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ vectorcraft-metafile = { path = "crates/metafile" } vectorcraft-plugins = { path = "crates/plugins" } vectorcraft-affinity = { path = "crates/affinity" } vectorcraft-scriptcheck = { path = "crates/scriptcheck" } +vectorcraft-js = { path = "crates/js" } # foundation kurbo = { version = "0.13", features = ["serde"] } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 6789a2116..eddf966db 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -66,3 +66,5 @@ vectorcraft-pdf = { workspace = true, features = ["read-count"] } wat = { workspace = true } # Zstandard streams for the Illustrator editing data fuzz tests. ruzstd = { workspace = true } +# The JavaScript parser's lexer, compared with the script checker's on the same scripts. +vectorcraft-js = { workspace = true } diff --git a/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs new file mode 100644 index 000000000..6c84c66b3 --- /dev/null +++ b/crates/engine/tests/it/js_lexer_vs_scriptcheck.rs @@ -0,0 +1,102 @@ +//! The JavaScript parser's lexer (`vectorcraft-js`) and the script checker's (`vectorcraft-scriptcheck`) +//! read the same scripts the same way: both crates stand alone, so each has its own lexer, and these +//! tests run both over one set of scripts and fail on any disagreement about the lines a script has, +//! its preprocessor lines, and where a string, comment or regular expression that is never closed +//! starts. The scripts stay inside ECMAScript 3, the language both read (no XML literals, which +//! only the checker reads). + +use vectorcraft_js::{Options, TokenKind, parse, tokenize}; +use vectorcraft_scriptcheck::{Limits, check}; + +/// The problems the checker reports from its lexer, which the parser reports as a `SyntaxError`. +const LEXICAL: [(&str, &str); 3] = [ + ("unterminatedString", "unterminated string"), + ("unterminatedComment", "unterminated comment"), + ("unterminatedRegExp", "unterminated regular expression"), +]; + +/// Scripts both lexers read without a problem, each with a `/` or a quote in a place where reading +/// it the other way would change the tokens. +const VALID: &[&str] = &[ + "var a = b / c / d;\nvar r = /ab+c/gi;\n", + "if (x) /re/.test(y);\nwhile (z) /q/g.exec(w);\n", + "function f() { return /x/; }\nvar g = function () {} / 2;\n", + "{}\n/re/.test(s);\n", + "x = {a: 1} / 2;\n", + "a = b\n/c/g\n", + "s = 'it\\'s \"q\" // not a comment';\nt = \"/* nor this */\";\n", + "s = 'line\\\ncontinued';\nu = 1;\n", + "x = a++ / 2;\ny = (b) / 2;\nz = c[0] / 2;\n", + "r = /[/]/;\nq = /\\//;\n", + "#target vectorcraft\n#include \"lib.jsxinc\"\n//@includepath \"~/x\"\nvar a = 1;\n#targetengine session\n", + "a = 1;\r\nb = 2;\r\nc = 3;", + "a = 1;\u{2028}b = 2;\u{2029}c = 3;\n", + "typeof /x/ === 'object';\nvoid /y/;\n", + "switch (x) { case /a/.test(y): break; }\n", + "x = y ? /a/ : /b/;\n", + "label: for (;;) { break label; }\n/* multi\nline */ z = 1 / 2;\n", + "var o = { re: /x/, d: 4 / 2 };\n", + "x = [/a/, 1 / 2];\n", + "if (a) {\n} else /x/.test(b);\n", + "a = 'q' + \"r\";\n// a comment with a 'quote\n", + "x = a\n++b\n", + "for (var i = 0; i < n / 2; i++) { s += /\\d/.exec(t[i]) ? 1 : 0; }\n", + "", + "\n\n", +]; + +/// Scripts with a string, comment or regular expression that is never closed: the problem the +/// checker gives and the line it starts on. +const INVALID: &[(&str, &str, usize)] = &[ + ("var a = 1;\nvar s = 'abc;\n", "unterminatedString", 2), + ("a = 1;\n/* never closed\nb = 2;\n", "unterminatedComment", 2), + ("x = 1;\ny = /abc\n", "unterminatedRegExp", 2), + ("if (x)\n /abc\n", "unterminatedRegExp", 2), + ("a = 1;\r\nb = \"x\r\n", "unterminatedString", 2), + ("q = 2 / 3;\nr = /[/]\n", "unterminatedRegExp", 2), +]; + +fn js_options() -> Options { + Options { preprocessor_lines: true, ..Options::default() } +} + +#[test] +fn both_lexers_count_the_same_lines_and_read_the_same_directives() { + for src in VALID { + let program = parse(src, &js_options()).unwrap_or_else(|e| panic!("{src:?}: {e}")); + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + assert_eq!(report.lines, program.line_count() as usize, "lines of {src:?}"); + let lexical: Vec<_> = report.problems.iter().filter(|p| LEXICAL.iter().any(|(code, _)| *code == p.code)).collect(); + assert!(lexical.is_empty(), "{src:?}: the checker reports {lexical:?}"); + let theirs: Vec<(String, usize)> = report.directives.iter().map(|d| (d.name.to_string(), d.line)).collect(); + let ours: Vec<(String, usize)> = program.directives.iter().map(|d| (d.name.to_lowercase(), d.line as usize)).collect(); + assert_eq!(ours, theirs, "directives of {src:?}"); + } +} + +#[test] +fn both_lexers_find_the_same_unclosed_token_on_the_same_line() { + for (src, code, line) in INVALID { + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + let theirs = report.problems.iter().find(|p| LEXICAL.iter().any(|(c, _)| *c == p.code)); + let theirs = theirs.unwrap_or_else(|| panic!("{src:?}: the checker reports no lexical problem")); + assert_eq!((theirs.code, theirs.line), (*code, Some(*line)), "{src:?}"); + let ours = parse(src, &js_options()).expect_err(src); + let message = LEXICAL.iter().find(|(c, _)| c == code).map(|(_, m)| *m).unwrap(); + assert!(ours.message.contains(message), "{src:?}: {}", ours.message); + assert_eq!(ours.line as usize, *line, "{src:?}"); + } +} + +#[test] +fn the_parser_reads_a_regexp_wherever_the_checker_does() { + // Each `/` that divides has no second `/` after it on its line, so a lexer that read one as a + // regular expression would report it as never closed. The checker reports nothing, and the + // parser reads regular expressions on lines 2 and 3 only. + let src = "a = b / c;\nr = /x/;\nif (a) /y/.test(b);\nz = (1) / 2;\n"; + let tokens = tokenize(src, &js_options()).unwrap(); + let lines: Vec = tokens.iter().filter(|t| matches!(t.kind, TokenKind::RegExp { .. })).map(|t| t.line).collect(); + assert_eq!(lines, [2, 3]); + let report = check("script.jsx", src.as_bytes(), None, &Limits::default()); + assert!(report.problems.iter().all(|p| !LEXICAL.iter().any(|(c, _)| *c == p.code)), "{:?}", report.problems); +} diff --git a/crates/engine/tests/it/main.rs b/crates/engine/tests/it/main.rs index a6f0b02c7..3689d9cae 100644 --- a/crates/engine/tests/it/main.rs +++ b/crates/engine/tests/it/main.rs @@ -18,6 +18,7 @@ mod dimensions; mod eyedropper_sources; mod find_replace_paras; mod inline_text; +mod js_lexer_vs_scriptcheck; mod known_bugs; mod lossy_overwrite; mod model_based; diff --git a/crates/js/Cargo.toml b/crates/js/Cargo.toml new file mode 100644 index 000000000..0108b50d9 --- /dev/null +++ b/crates/js/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vectorcraft-js" +description = "JavaScript for scripts: an ECMAScript 5.1 lexer and parser to a syntax tree with source positions" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +unicode-general-category = "1" + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/js/src/ast.rs b/crates/js/src/ast.rs new file mode 100644 index 000000000..f0dd6b44b --- /dev/null +++ b/crates/js/src/ast.rs @@ -0,0 +1,334 @@ +//! The syntax tree: expressions, statements and functions kept in arenas ([`Ast`]) and referred to +//! by index, each with its source span. Nodes hold indices rather than boxes, so dropping a deeply +//! nested tree does not recurse. + +use crate::token::Span; + +/// An expression's index in [`Ast::exprs`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ExprId(pub u32); + +/// A statement's index in [`Ast::stmts`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct StmtId(pub u32); + +/// A function's index in [`Ast::functions`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FunctionId(pub u32); + +/// An identifier as written, with its span. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Name { + pub name: String, + pub span: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UnaryOp { + Delete, + Void, + Typeof, + Plus, + Minus, + BitNot, + Not, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UpdateOp { + Increment, + Decrement, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BinaryOp { + Mul, + Div, + Mod, + Add, + Sub, + Shl, + Shr, + UShr, + Lt, + Gt, + LtEq, + GtEq, + Instanceof, + In, + Eq, + NotEq, + StrictEq, + StrictNotEq, + BitAnd, + BitXor, + BitOr, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LogicalOp { + And, + Or, +} + +/// `=` (`None`) or a compound assignment's operator (`+=` is `Some(BinaryOp::Add)`). +pub type AssignOp = Option; + +/// A property name in an object literal. +#[derive(Clone, Debug, PartialEq)] +pub enum PropertyKey { + Ident(String), + String(Vec), + Number(f64), +} + +impl PropertyKey { + /// The key as the string a property is stored under (a number through ToString). + pub fn to_utf16(&self) -> Vec { + match self { + PropertyKey::Ident(name) => name.encode_utf16().collect(), + PropertyKey::String(s) => s.clone(), + PropertyKey::Number(n) => crate::number::to_string(*n).encode_utf16().collect(), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum PropertyValue { + Init(ExprId), + Get(FunctionId), + Set(FunctionId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Property { + pub key: PropertyKey, + pub value: PropertyValue, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Expr { + This, + Ident(String), + Null, + Bool(bool), + Number(f64), + String(Vec), + RegExp { + pattern: Vec, + flags: String, + }, + /// An array literal; `None` is a hole (`[1, , 3]`). + Array(Vec>), + Object(Vec), + Function(FunctionId), + /// `object.property`. + Member { + object: ExprId, + property: Name, + }, + /// `object[index]`. + Index { + object: ExprId, + index: ExprId, + }, + Call { + callee: ExprId, + args: Vec, + }, + New { + callee: ExprId, + args: Vec, + }, + Update { + op: UpdateOp, + prefix: bool, + target: ExprId, + }, + Unary { + op: UnaryOp, + arg: ExprId, + }, + Binary { + op: BinaryOp, + left: ExprId, + right: ExprId, + }, + Logical { + op: LogicalOp, + left: ExprId, + right: ExprId, + }, + Conditional { + test: ExprId, + consequent: ExprId, + alternate: ExprId, + }, + Assign { + op: AssignOp, + target: ExprId, + value: ExprId, + }, + /// Comma-separated expressions. + Sequence(Vec), + /// A parenthesized expression, kept so a rule about the bare form (a label, a directive) can + /// tell them apart. + Paren(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct VarDecl { + pub name: Name, + pub init: Option, +} + +/// What a `for (… in …)` statement assigns each property name to. +#[derive(Clone, Debug, PartialEq)] +pub enum ForInTarget { + /// `for (var x in o)`, with the declaration's initializer if it has one. + Var(VarDecl), + Expr(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ForInit { + Var(Vec), + Expr(ExprId), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SwitchCase { + /// `None` for `default`. + pub test: Option, + pub body: Vec, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CatchClause { + pub param: Name, + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Stmt { + Var(Vec), + FunctionDecl(FunctionId), + Expr(ExprId), + Block(Vec), + Empty, + If { test: ExprId, consequent: StmtId, alternate: Option }, + DoWhile { body: StmtId, test: ExprId }, + While { test: ExprId, body: StmtId }, + For { init: Option, test: Option, update: Option, body: StmtId }, + ForIn { target: ForInTarget, object: ExprId, body: StmtId }, + Continue(Option), + Break(Option), + Return(Option), + With { object: ExprId, body: StmtId }, + Switch { discriminant: ExprId, cases: Vec }, + Labeled { label: Name, body: StmtId }, + Throw(ExprId), + Try { block: Vec, handler: Option, finalizer: Option> }, + Debugger, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FunctionKind { + Declaration, + Expression, + Getter, + Setter, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Function { + pub kind: FunctionKind, + pub name: Option, + pub params: Vec, + pub body: Vec, + /// The function's code is strict mode code: its own directive prologue holds `"use strict"`, + /// or the code around it is strict. + pub strict: bool, + pub span: Span, +} + +/// The arenas of a parsed script. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Ast { + pub exprs: Vec<(Expr, Span)>, + pub stmts: Vec<(Stmt, Span)>, + pub functions: Vec, +} + +impl Ast { + pub fn expr(&self, id: ExprId) -> Option<&Expr> { + self.exprs.get(id.0 as usize).map(|(e, _)| e) + } + + pub fn expr_span(&self, id: ExprId) -> Span { + self.exprs.get(id.0 as usize).map(|(_, s)| *s).unwrap_or_default() + } + + pub fn stmt(&self, id: StmtId) -> Option<&Stmt> { + self.stmts.get(id.0 as usize).map(|(s, _)| s) + } + + pub fn stmt_span(&self, id: StmtId) -> Span { + self.stmts.get(id.0 as usize).map(|(_, s)| *s).unwrap_or_default() + } + + pub fn function(&self, id: FunctionId) -> Option<&Function> { + self.functions.get(id.0 as usize) + } + + /// The expression with any parentheses around it removed. + pub fn unparenthesized(&self, mut id: ExprId) -> ExprId { + while let Some(Expr::Paren(inner)) = self.expr(id) { + id = *inner; + } + id + } + + pub(crate) fn push_expr(&mut self, expr: Expr, span: Span) -> ExprId { + self.exprs.push((expr, span)); + ExprId(u32::try_from(self.exprs.len() - 1).unwrap_or(u32::MAX)) + } + + pub(crate) fn push_stmt(&mut self, stmt: Stmt, span: Span) -> StmtId { + self.stmts.push((stmt, span)); + StmtId(u32::try_from(self.stmts.len() - 1).unwrap_or(u32::MAX)) + } + + pub(crate) fn push_function(&mut self, function: Function) -> FunctionId { + self.functions.push(function); + FunctionId(u32::try_from(self.functions.len() - 1).unwrap_or(u32::MAX)) + } +} + +/// A parsed script. +#[derive(Clone, Debug, PartialEq)] +pub struct Program { + pub ast: Ast, + pub body: Vec, + /// The script is strict mode code (its directive prologue holds `"use strict"`, or it was + /// parsed with [`crate::Options::strict`]). + pub strict: bool, + /// Its preprocessor lines, in source order. + pub directives: Vec, + pub(crate) lines: crate::LineTable, +} + +impl Program { + /// The line and column (both from 1, the column in UTF-16 code units) of a source offset. + pub fn line_column(&self, offset: u32) -> (u32, u32) { + self.lines.line_column(offset) + } + + /// The number of lines in the source: its line terminators plus one for a last line without + /// one. + pub fn line_count(&self) -> u32 { + self.lines.line_count() + } +} diff --git a/crates/js/src/chars.rs b/crates/js/src/chars.rs new file mode 100644 index 000000000..75c0b92aa --- /dev/null +++ b/crates/js/src/chars.rs @@ -0,0 +1,67 @@ +//! Character classes of ECMA-262 5.1 section 7: white space, line terminators and the characters +//! identifiers are made of. + +use unicode_general_category::{GeneralCategory as G, get_general_category}; + +/// A line terminator: LF, CR, U+2028 or U+2029 (section 7.3). +pub fn is_line_terminator(c: u32) -> bool { + matches!(c, 0x0A | 0x0D | 0x2028 | 0x2029) +} + +/// White space (section 7.2): tab, vertical tab, form feed, space, no-break space, the byte order +/// mark and the other space separators (Unicode category Zs). +pub fn is_whitespace(c: u32) -> bool { + matches!(c, 0x09 | 0x0B | 0x0C | 0x20 | 0xA0 | 0xFEFF | 0x1680 | 0x2000..=0x200A | 0x202F | 0x205F | 0x3000) +} + +/// A character an identifier may start with: a letter (categories Lu, Ll, Lt, Lm, Lo and Nl), +/// `$` or `_` (section 7.6). `\u` escapes are decoded before this check. +pub fn is_id_start(c: u32) -> bool { + match c { + 0x24 | 0x5F => true, + 0x41..=0x5A | 0x61..=0x7A => true, + 0..=0x7F => false, + // U+2E2F VERTICAL TILDE is a letter (Lm) that Unicode leaves out of identifiers. + 0x2E2F => false, + _ => category(c).is_some_and(|g| { + matches!(g, G::UppercaseLetter | G::LowercaseLetter | G::TitlecaseLetter | G::ModifierLetter | G::OtherLetter | G::LetterNumber) + }), + } +} + +/// A character an identifier may hold after its first: those it may start with, decimal digits +/// (Nd), combining marks (Mn, Mc), connector punctuation (Pc), and the zero width non-joiner and +/// joiner. +pub fn is_id_part(c: u32) -> bool { + match c { + 0x30..=0x39 => true, + 0x200C | 0x200D => true, + 0x2E2F => false, + _ if is_id_start(c) => true, + 0..=0x7F => false, + _ => category(c).is_some_and(|g| matches!(g, G::NonspacingMark | G::SpacingMark | G::DecimalNumber | G::ConnectorPunctuation)), + } +} + +/// The Unicode general category of a code point; `None` for a lone surrogate. +fn category(c: u32) -> Option { + char::from_u32(c).map(get_general_category) +} + +/// The value of a hexadecimal digit. +pub fn hex_value(c: u32) -> Option { + match c { + 0x30..=0x39 => Some(c - 0x30), + 0x41..=0x46 => Some(c - 0x41 + 10), + 0x61..=0x66 => Some(c - 0x61 + 10), + _ => None, + } +} + +pub fn is_decimal_digit(c: u32) -> bool { + (0x30..=0x39).contains(&c) +} + +pub fn is_octal_digit(c: u32) -> bool { + (0x30..=0x37).contains(&c) +} diff --git a/crates/js/src/error.rs b/crates/js/src/error.rs new file mode 100644 index 000000000..9c17bf8be --- /dev/null +++ b/crates/js/src/error.rs @@ -0,0 +1,29 @@ +//! The error a script that cannot be read gives: a `SyntaxError` with its message and where in the +//! source it is. + +use std::fmt; + +/// A script that does not follow the grammar, or breaks one of its early-error rules. `line` and +/// `column` count from 1; the column counts UTF-16 code units from the start of the line, as +/// JavaScript source positions do. `offset` is the UTF-16 offset from the start of the source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SyntaxError { + pub message: String, + pub line: u32, + pub column: u32, + pub offset: u32, +} + +impl fmt::Display for SyntaxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SyntaxError: {} (line {}, column {})", self.message, self.line, self.column) + } +} + +impl std::error::Error for SyntaxError {} + +/// The message of the error nesting deeper than [`crate::Options::max_depth`] gives. +pub const TOO_DEEP: &str = "nesting too deep"; + +/// The message of the error a source longer than [`crate::Options::max_source_units`] gives. +pub const TOO_LONG: &str = "source too long"; diff --git a/crates/js/src/lexer.rs b/crates/js/src/lexer.rs new file mode 100644 index 000000000..9b16b16c7 --- /dev/null +++ b/crates/js/src/lexer.rs @@ -0,0 +1,637 @@ +//! The lexer (ECMA-262 5.1 section 7): source text in UTF-16 code units into tokens, with each +//! token's line and column and whether a line terminator comes before it. +//! +//! A `/` starts a division punctuator or a regular expression literal depending on where the +//! grammar is (section 7, the InputElementDiv and InputElementRegExp goals). The lexer reads a +//! token for the goal it is given; the parser reads tokens for the division goal and asks for a +//! token to be read again as a regular expression where an expression starts +//! ([`Lexer::rescan_regexp`]). +//! +//! With [`crate::Options::preprocessor_lines`], a `#name value` line, or a `//@name value` comment +//! that names a directive, at the start of a line (only blanks before it) gives a +//! [`TokenKind::Directive`] token. +//! +//! The lexer moves forward through the source one character at a time and never recurses. + +use crate::chars::{hex_value, is_decimal_digit, is_id_part, is_id_start, is_line_terminator, is_octal_digit, is_whitespace}; +use crate::error::SyntaxError; +use crate::regexp; +use crate::token::{Directive, DirectiveName, Keyword, Punct, Span, Token, TokenKind}; + +/// Which goal symbol the next token is read for: whether a `/` there divides or starts a regular +/// expression literal. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Goal { + Div, + RegExp, +} + +const BACKSLASH: u32 = b'\\' as u32; +const SLASH: u32 = b'/' as u32; +const STAR: u32 = b'*' as u32; +const DOT: u32 = b'.' as u32; +const CR: u32 = 0x0D; +const LF: u32 = 0x0A; + +/// The lexer's state. It is small and cheap to clone, which is how the parser looks one token +/// ahead. +#[derive(Clone, Debug)] +pub struct Lexer<'s> { + src: &'s [u16], + pos: usize, + /// The current line (from 1) and the offset where it starts. + line: u32, + line_start: usize, + /// Only blanks since the start of the line: a preprocessor line may start here. + blank_line: bool, + preprocessor_lines: bool, + /// The deepest nesting of groups a regular expression literal may have. + max_depth: usize, +} + +impl<'s> Lexer<'s> { + /// A lexer of `src`. `preprocessor_lines` turns on the reading of preprocessor lines; + /// `max_depth` limits how deeply a regular expression literal's groups may nest. + pub fn new(src: &'s [u16], preprocessor_lines: bool, max_depth: usize) -> Self { + Self { src, pos: 0, line: 1, line_start: 0, blank_line: true, preprocessor_lines, max_depth } + } + + /// The code unit at `pos`, as a code point value. + fn unit(&self, pos: usize) -> Option { + self.src.get(pos).map(|&u| u32::from(u)) + } + + fn peek(&self) -> Option { + self.unit(self.pos) + } + + fn peek_at(&self, n: usize) -> Option { + self.unit(self.pos + n) + } + + /// The code point at `pos` (a surrogate pair read as one) and how many units it takes. + fn code_point(&self, pos: usize) -> Option<(u32, usize)> { + let hi = self.unit(pos)?; + if (0xD800..0xDC00).contains(&hi) + && let Some(lo) = self.unit(pos + 1).filter(|lo| (0xDC00..0xE000).contains(lo)) + { + return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), 2)); + } + Some((hi, 1)) + } + + fn column_of(&self, pos: usize) -> u32 { + u32::try_from(pos.saturating_sub(self.line_start) + 1).unwrap_or(u32::MAX) + } + + fn offset(pos: usize) -> u32 { + u32::try_from(pos).unwrap_or(u32::MAX) + } + + /// An error at `pos` on the current line. + fn error_at(&self, pos: usize, message: impl Into) -> SyntaxError { + SyntaxError { message: message.into(), line: self.line, column: self.column_of(pos), offset: Self::offset(pos) } + } + + /// An error at a position recorded earlier. + fn error_from(start: (usize, u32, u32), message: impl Into) -> SyntaxError { + SyntaxError { message: message.into(), line: start.1, column: start.2, offset: Self::offset(start.0) } + } + + /// Reads a line terminator at `pos` (CR LF as one), counting the line. False when there is + /// none. + fn line_terminator(&mut self) -> bool { + match self.peek() { + Some(CR) if self.peek_at(1) == Some(LF) => self.pos += 2, + Some(c) if is_line_terminator(c) => self.pos += 1, + _ => return false, + } + self.line = self.line.saturating_add(1); + self.line_start = self.pos; + self.blank_line = true; + true + } + + /// The next token, read for `goal`. + pub fn next_token(&mut self, goal: Goal) -> Result { + let newline_before = match self.skip_trivia()? { + Trivia::Directive(token) => return Ok(*token), + Trivia::Skipped { newline } => newline, + }; + let start = self.pos; + let (line, column) = (self.line, self.column_of(start)); + let Some(c) = self.peek() else { + return Ok(self.token(TokenKind::Eof, start, line, column, newline_before)); + }; + self.blank_line = false; + let mut token = if c == BACKSLASH || is_id_start(c) || self.code_point(start).is_some_and(|(cp, n)| n == 2 && is_id_start(cp)) { + self.identifier(start, line, column)? + } else if is_decimal_digit(c) || (c == DOT && self.peek_at(1).is_some_and(is_decimal_digit)) { + self.number(start, line, column)? + } else if c == u32::from(b'"') || c == u32::from(b'\'') { + self.string(start, line, column)? + } else if c == SLASH && goal == Goal::RegExp { + self.regexp(start, line, column)? + } else { + let punct = self.punct().ok_or_else(|| self.error_at(start, unexpected_character(c)))?; + self.token(TokenKind::Punct(punct), start, line, column, false) + }; + // A string continued over a line ends on a new line, but not at its start. + self.blank_line = false; + token.newline_before = newline_before; + Ok(token) + } + + /// Reads the token `token` again as a regular expression literal: the parser read a `/` or + /// `/=` where an expression starts. + pub fn rescan_regexp(&mut self, token: &Token) -> Result { + let start = usize::try_from(token.span.start).unwrap_or(usize::MAX); + self.pos = start; + self.line = token.line; + self.line_start = start.saturating_sub(usize::try_from(token.column.saturating_sub(1)).unwrap_or(0)); + self.blank_line = false; + let mut regexp = self.regexp(start, token.line, token.column)?; + regexp.newline_before = token.newline_before; + Ok(regexp) + } + + fn token(&self, kind: TokenKind, start: usize, line: u32, column: u32, newline_before: bool) -> Token { + Token { + kind, + span: Span::new(Self::offset(start), Self::offset(self.pos)), + line, + column, + newline_before, + escaped: false, + legacy_octal: false, + } + } + + /// Skips white space, line terminators and comments, and reads a preprocessor line when one + /// starts here. + fn skip_trivia(&mut self) -> Result { + let mut newline = false; + while let Some(c) = self.peek() { + if is_whitespace(c) { + self.pos += 1; + } else if self.line_terminator() { + newline = true; + } else if c == SLASH && self.peek_at(1) == Some(SLASH) { + if self.blank_line && self.preprocessor_lines && self.names_directive(self.pos + 2) { + return Ok(Trivia::Directive(Box::new(self.directive("//@", newline)))); + } + while self.peek().is_some_and(|c| !is_line_terminator(c)) { + self.pos += 1; + } + } else if c == SLASH && self.peek_at(1) == Some(STAR) { + newline |= self.block_comment()?; + // The comment's end is on this line, so a `#` after it is not at the line's start. + self.blank_line = false; + } else if c == u32::from(b'#') && self.blank_line && self.preprocessor_lines && self.peek_at(1).is_some_and(is_name_char) { + return Ok(Trivia::Directive(Box::new(self.directive("#", newline)))); + } else { + break; + } + } + Ok(Trivia::Skipped { newline }) + } + + /// Skips a `/* */` comment; true when it holds a line terminator. + fn block_comment(&mut self) -> Result { + let start = (self.pos, self.line, self.column_of(self.pos)); + self.pos += 2; + let mut newline = false; + loop { + match self.peek() { + None => return Err(Self::error_from(start, "unterminated comment")), + Some(STAR) if self.peek_at(1) == Some(SLASH) => { + self.pos += 2; + return Ok(newline); + } + Some(_) if self.line_terminator() => newline = true, + Some(_) => self.pos += 1, + } + } + } + + /// `//@` at `pos - 2` is followed by a directive's name. + fn names_directive(&self, pos: usize) -> bool { + if self.unit(pos) != Some(u32::from(b'@')) { + return false; + } + let name: String = self.src[pos + 1..].iter().take_while(|&&u| is_name_char(u32::from(u))).map(|&u| char::from(u as u8)).collect(); + DirectiveName::from_name(&name).is_some() + } + + /// Reads a preprocessor line written with `form` (`#` or `//@`), up to the end of the line. + fn directive(&mut self, form: &'static str, newline_before: bool) -> Token { + let start = self.pos; + let (line, column) = (self.line, self.column_of(start)); + self.pos += form.encode_utf16().count(); + let name_start = self.pos; + while self.peek().is_some_and(is_name_char) { + self.pos += 1; + } + let name = String::from_utf16_lossy(&self.src[name_start..self.pos]); + let value_start = self.pos; + while self.peek().is_some_and(|c| !is_line_terminator(c)) { + self.pos += 1; + } + let value = String::from_utf16_lossy(&self.src[value_start..self.pos]).trim().to_string(); + let directive = Directive { + known: DirectiveName::from_name(&name), + name, + value, + form, + span: Span::new(Self::offset(start), Self::offset(self.pos)), + line, + }; + self.blank_line = false; + self.token(TokenKind::Directive(directive), start, line, column, newline_before) + } + + /// Reads an identifier or a keyword (section 7.6), `\u` escapes included. + fn identifier(&mut self, start: usize, line: u32, column: u32) -> Result { + let mut name = String::new(); + let mut escaped = false; + let mut first = true; + loop { + let at = self.pos; + let (c, len, is_escape) = match self.code_point(at) { + Some((BACKSLASH, _)) => { + let c = self.identifier_escape()?; + (c, 0, true) + } + Some((c, len)) => (c, len, false), + None => break, + }; + let ok = if first { is_id_start(c) } else { is_id_part(c) }; + if !ok { + if is_escape { + return Err(self.error_at(at, "invalid character in identifier escape")); + } + break; + } + self.pos += len; + escaped |= is_escape; + first = false; + match char::from_u32(c) { + Some(ch) => name.push(ch), + None => return Err(self.error_at(at, "invalid character in identifier")), + } + } + let kind = match Keyword::from_word(&name) { + Some(k) if !escaped => TokenKind::Keyword(k), + _ => TokenKind::Ident(name), + }; + let mut token = self.token(kind, start, line, column, false); + token.escaped = escaped; + Ok(token) + } + + /// Reads a `\uXXXX` escape in an identifier and returns its code point. + fn identifier_escape(&mut self) -> Result { + let at = self.pos; + if self.peek_at(1) != Some(u32::from(b'u')) { + return Err(self.error_at(at, "invalid escape in identifier")); + } + self.pos += 2; + self.unicode_escape().ok_or_else(|| self.error_at(at, "invalid Unicode escape sequence")) + } + + /// After `\u`: four hexadecimal digits, or a code point in braces (`\u{1F600}`, ECMA-262 2015 + /// section 11.8.4) up to U+10FFFF. + fn unicode_escape(&mut self) -> Option { + if self.peek() != Some(u32::from(b'{')) { + return self.hex_digits(4); + } + let mut n = 1; + let mut value: u32 = 0; + while let Some(d) = self.peek_at(n).and_then(hex_value) { + value = value.checked_mul(16)?.checked_add(d).filter(|&v| v <= 0x10FFFF)?; + n += 1; + } + if n == 1 || self.peek_at(n) != Some(u32::from(b'}')) { + return None; + } + self.pos += n + 1; + Some(value) + } + + /// Reads `n` hexadecimal digits and returns their value, or `None` (leaving the position) when + /// fewer follow. + fn hex_digits(&mut self, n: usize) -> Option { + let mut value = 0; + for i in 0..n { + value = value * 16 + hex_value(self.peek_at(i)?)?; + } + self.pos += n; + Some(value) + } + + /// Reads a numeric literal (section 7.8.3, with the legacy octal literals of Annex B). + fn number(&mut self, start: usize, line: u32, column: u32) -> Result { + let mut legacy_octal = false; + let value = if self.peek() == Some(u32::from(b'0')) && self.peek_at(1).is_some_and(|c| c == u32::from(b'x') || c == u32::from(b'X')) { + self.pos += 2; + self.radix_digits(16).ok_or_else(|| self.error_at(start, "hexadecimal literal without digits"))? + } else if self.peek() == Some(u32::from(b'0')) && self.peek_at(1).is_some_and(is_decimal_digit) { + legacy_octal = true; + self.legacy_octal_or_decimal() + } else { + self.decimal() + }; + if self.peek().is_some_and(|c| c == BACKSLASH || is_id_start(c) || is_decimal_digit(c)) { + return Err(self.error_at(self.pos, "identifier starts immediately after numeric literal")); + } + let mut token = self.token(TokenKind::Number(value), start, line, column, false); + token.legacy_octal = legacy_octal; + Ok(token) + } + + /// Reads digits in `radix` (8 or 16) and returns their value, or `None` when there are none. + fn radix_digits(&mut self, radix: u32) -> Option { + let begin = self.pos; + let mut exact: u128 = 0; + let mut value = 0.0_f64; + let mut overflow = false; + while let Some(d) = self.peek().and_then(hex_value).filter(|&d| d < radix) { + match exact.checked_mul(u128::from(radix)).and_then(|v| v.checked_add(u128::from(d))) { + Some(v) if !overflow => exact = v, + _ => { + if !overflow { + value = exact as f64; + overflow = true; + } + value = value * f64::from(radix) + f64::from(d); + } + } + self.pos += 1; + } + if self.pos == begin { + return None; + } + Some(if overflow { value } else { exact as f64 }) + } + + /// After `0` and a digit: an octal literal when every digit is octal, else a decimal literal + /// whose integer part starts with `0` (`08`, `09.5`). + fn legacy_octal_or_decimal(&mut self) -> f64 { + let begin = self.pos; + let mut end = begin; + while self.unit(end).is_some_and(is_decimal_digit) { + end += 1; + } + if self.src[begin..end].iter().all(|&u| is_octal_digit(u32::from(u))) { + self.pos += 1; + return self.radix_digits(8).unwrap_or(0.0); + } + self.decimal() + } + + /// Reads a decimal literal: digits, an optional fraction and an optional exponent. + fn decimal(&mut self) -> f64 { + let mut text = String::new(); + self.push_digits(&mut text); + if text.is_empty() { + text.push('0'); + } + if self.peek() == Some(DOT) { + self.pos += 1; + text.push('.'); + let before = text.len(); + self.push_digits(&mut text); + if text.len() == before { + text.push('0'); + } + } + if self.peek().is_some_and(|c| c == u32::from(b'e') || c == u32::from(b'E')) { + let sign = self.peek_at(1).filter(|&c| c == u32::from(b'+') || c == u32::from(b'-')); + let digit_at = if sign.is_some() { 2 } else { 1 }; + if self.peek_at(digit_at).is_some_and(is_decimal_digit) { + self.pos += digit_at; + text.push('e'); + if sign == Some(u32::from(b'-')) { + text.push('-'); + } + self.push_digits(&mut text); + } + } + text.parse().unwrap_or(f64::NAN) + } + + fn push_digits(&mut self, text: &mut String) { + while let Some(c) = self.peek().filter(|&c| is_decimal_digit(c)) { + text.push(char::from(c as u8)); + self.pos += 1; + } + } + + /// Reads a string literal (section 7.8.4, with the octal escapes of Annex B) and returns its + /// value. + fn string(&mut self, start: usize, line: u32, column: u32) -> Result { + let quote = self.peek().unwrap_or_default(); + self.pos += 1; + let mut value = Vec::new(); + let mut legacy_octal = false; + loop { + match self.peek() { + None => return Err(Self::error_from((start, line, column), "unterminated string literal")), + // U+2028 and U+2029 may stand in a string, as ECMA-262 2019 allows; they still + // start a new line for positions. + Some(c @ (0x2028 | 0x2029)) => { + value.push(c as u16); + self.line_terminator(); + } + Some(c) if is_line_terminator(c) => { + return Err(Self::error_from((start, line, column), "unterminated string literal")); + } + Some(c) if c == quote => { + self.pos += 1; + break; + } + Some(BACKSLASH) => legacy_octal |= self.string_escape(&mut value)?, + Some(c) => { + value.push(c as u16); + self.pos += 1; + } + } + } + let mut token = self.token(TokenKind::String(value), start, line, column, false); + token.legacy_octal = legacy_octal; + Ok(token) + } + + /// Reads an escape in a string literal into `value`; true for an octal escape (`\1`, `\08`) + /// or `\8` and `\9`. + fn string_escape(&mut self, value: &mut Vec) -> Result { + let at = self.pos; + self.pos += 1; + let Some(c) = self.peek() else { + return Err(self.error_at(at, "unterminated string literal")); + }; + if self.line_terminator() { + return Ok(false); + } + let single = match char::from_u32(c) { + Some('b') => Some(0x08), + Some('t') => Some(0x09), + Some('n') => Some(0x0A), + Some('v') => Some(0x0B), + Some('f') => Some(0x0C), + Some('r') => Some(0x0D), + _ => None, + }; + if let Some(unit) = single { + value.push(unit); + self.pos += 1; + return Ok(false); + } + match char::from_u32(c) { + Some('x') => { + self.pos += 1; + let unit = self.hex_digits(2).ok_or_else(|| self.error_at(at, "invalid hexadecimal escape sequence"))?; + value.push(unit as u16); + Ok(false) + } + Some('u') => { + self.pos += 1; + let code_point = self.unicode_escape().ok_or_else(|| self.error_at(at, "invalid Unicode escape sequence"))?; + match char::from_u32(code_point) { + Some(ch) if code_point > 0xFFFF => value.extend(ch.encode_utf16(&mut [0; 2]).iter()), + _ => value.push(code_point as u16), + } + Ok(false) + } + Some('0') if !self.peek_at(1).is_some_and(is_decimal_digit) => { + self.pos += 1; + value.push(0); + Ok(false) + } + Some('0'..='7') => { + value.push(self.octal_escape(c)); + Ok(true) + } + Some('8' | '9') => { + self.pos += 1; + value.push(c as u16); + Ok(true) + } + _ => { + self.pos += 1; + value.push(c as u16); + Ok(false) + } + } + } + + /// Reads an octal escape whose first digit is `first`: up to three digits when it is 0 to 3, + /// up to two when it is 4 to 7. + fn octal_escape(&mut self, first: u32) -> u16 { + let max = if first <= u32::from(b'3') { 3 } else { 2 }; + let mut value = 0; + let mut n = 0; + while n < max + && let Some(d) = self.peek().filter(|&d| is_octal_digit(d)) + { + value = value * 8 + (d - u32::from(b'0')); + self.pos += 1; + n += 1; + } + value as u16 + } + + /// Reads a regular expression literal (section 7.8.5) and checks its flags and pattern. + fn regexp(&mut self, start: usize, line: u32, column: u32) -> Result { + let at = (start, line, column); + self.pos = start + 1; + let mut in_class = false; + loop { + match self.peek() { + None => return Err(Self::error_from(at, "unterminated regular expression literal")), + Some(c) if is_line_terminator(c) => return Err(Self::error_from(at, "unterminated regular expression literal")), + Some(BACKSLASH) => { + if self.peek_at(1).is_none_or(is_line_terminator) { + return Err(Self::error_from(at, "unterminated regular expression literal")); + } + self.pos += 2; + } + Some(c) if c == u32::from(b'[') => { + in_class = true; + self.pos += 1; + } + Some(c) if c == u32::from(b']') => { + in_class = false; + self.pos += 1; + } + Some(SLASH) if !in_class => break, + Some(_) => self.pos += 1, + } + } + let pattern = self.src[start + 1..self.pos].to_vec(); + self.pos += 1; + let mut flags = String::new(); + while let Some((c, len)) = self.code_point(self.pos) { + if c == BACKSLASH { + return Err(Self::error_from(at, "invalid regular expression flags")); + } + if !is_id_part(c) { + break; + } + flags.push(char::from_u32(c).unwrap_or('\u{fffd}')); + self.pos += len; + } + if !valid_flags(&flags) { + return Err(Self::error_from(at, "invalid regular expression flags")); + } + if let Err(message) = regexp::check(&pattern, self.max_depth) { + return Err(Self::error_from(at, format!("invalid regular expression: {message}"))); + } + Ok(self.token(TokenKind::RegExp { pattern, flags }, start, line, column, false)) + } + + /// Reads the longest punctuator that starts here. + fn punct(&mut self) -> Option { + let rest = &self.src[self.pos..]; + for (spelling, punct) in Punct::ALL { + let bytes = spelling.as_bytes(); + if rest.len() >= bytes.len() && rest.iter().zip(bytes).all(|(&u, &b)| u == u16::from(b)) { + self.pos += bytes.len(); + return Some(*punct); + } + } + None + } +} + +/// What skipping the blanks and comments before a token found. +enum Trivia { + Skipped { newline: bool }, + Directive(Box), +} + +/// A character of a directive's name. +fn is_name_char(c: u32) -> bool { + c < 0x80 && (char::from(c as u8).is_ascii_alphanumeric() || c == u32::from(b'_')) +} + +/// The flags `g`, `i` and `m`, each at most once (section 15.10.4.1). +fn valid_flags(flags: &str) -> bool { + let mut seen = [false; 3]; + flags.chars().all(|c| { + let i = match c { + 'g' => 0, + 'i' => 1, + 'm' => 2, + _ => return false, + }; + !std::mem::replace(&mut seen[i], true) + }) +} + +fn unexpected_character(c: u32) -> String { + match char::from_u32(c) { + Some(ch) if !ch.is_control() => format!("unexpected character '{ch}'"), + _ => format!("unexpected character U+{c:04X}"), + } +} diff --git a/crates/js/src/lib.rs b/crates/js/src/lib.rs new file mode 100644 index 000000000..4d8d32271 --- /dev/null +++ b/crates/js/src/lib.rs @@ -0,0 +1,149 @@ +//! JavaScript for VectorCraft's scripts: an ECMAScript 5.1 lexer and parser +//! (), with the legacy forms of Annex B that older scripts +//! use (octal literals and escapes outside strict mode, function declarations in blocks, the +//! lenient regular expression patterns). +//! +//! [`parse`] reads a script into a [`Program`]: a syntax tree whose expressions, statements and +//! functions live in arenas and carry their source spans. A script that breaks the grammar or one +//! of its early-error rules (section 16: strict mode's restrictions, duplicate parameters in strict +//! code, invalid assignment targets, `break` and `continue` without a target, regular expression +//! literals that are not valid) gives a [`SyntaxError`] with its line and column. +//! +//! With [`Options::preprocessor_lines`], the lexer also reads the preprocessor lines scripts begin +//! with (`#include`, `#includepath`, `#script`, `#strict`, `#target`, `#targetengine`) into +//! [`Directive`]s. They are kept in [`Program::directives`] and never run. +//! +//! The lexer reads the source as UTF-16 code units, as JavaScript does: columns and spans count +//! code units. Nesting is counted: input nested deeper than [`Options::max_depth`] gives a +//! `SyntaxError` ("nesting too deep") instead of exhausting the stack, binary operators, `if` … +//! `else if` chains, labels and prefix operators are read with loops, and the syntax tree is +//! dropped without recursion. +#![forbid(unsafe_code)] + +pub mod ast; +mod chars; +mod error; +mod lexer; +mod number; +mod parser; +mod regexp; +mod token; + +pub use ast::{ + AssignOp, Ast, BinaryOp, CatchClause, Expr, ExprId, ForInTarget, ForInit, Function, FunctionId, FunctionKind, LogicalOp, Name, Program, Property, + PropertyKey, PropertyValue, Stmt, StmtId, SwitchCase, UnaryOp, UpdateOp, VarDecl, +}; +pub use error::{SyntaxError, TOO_DEEP, TOO_LONG}; +pub use lexer::{Goal, Lexer}; +pub use number::to_string as number_to_string; +pub use token::{Directive, DirectiveName, Keyword, Punct, Span, Token, TokenKind}; + +/// How a script is read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Options { + /// The script is strict mode code from its start, as code passed to a strict `eval` is. + pub strict: bool, + /// Read preprocessor lines (`#target …` and the other directives) into [`Directive`]s. Off, + /// a `#` is an unexpected character, as ECMA-262 has it. + pub preprocessor_lines: bool, + /// How deeply statements, expressions, functions and regular expression groups may nest. + pub max_depth: usize, + /// The most UTF-16 code units a source may hold. + pub max_source_units: usize, +} + +impl Default for Options { + fn default() -> Self { + Self { strict: false, preprocessor_lines: false, max_depth: DEFAULT_MAX_DEPTH, max_source_units: 64 << 20 } + } +} + +/// The default [`Options::max_depth`]: deeper than scripts nest, and shallow enough for the stack +/// of a wasm build or a secondary thread. +pub const DEFAULT_MAX_DEPTH: usize = 200; + +/// Parses `source` as a script. +pub fn parse(source: &str, options: &Options) -> Result { + let units = to_utf16(source, options)?; + parse_utf16(&units, options) +} + +/// Parses `source`, in UTF-16 code units, as a script. +pub fn parse_utf16(source: &[u16], options: &Options) -> Result { + check_length(source.len(), options)?; + parser::Parser::new(source, options, false)?.parse_program().map(|(program, _)| program) +} + +/// The tokens of `source`, in order, as the parser reads them: preprocessor lines included, and +/// each `/` read as division or as the start of a regular expression literal as the grammar +/// decides. The source must be a valid script. +pub fn tokenize(source: &str, options: &Options) -> Result, SyntaxError> { + let units = to_utf16(source, options)?; + check_length(units.len(), options)?; + parser::Parser::new(&units, options, true)?.parse_program().map(|(_, tokens)| tokens) +} + +fn to_utf16(source: &str, options: &Options) -> Result, SyntaxError> { + // A source's UTF-16 length is at most its UTF-8 length; count only a source that may be too + // long. + if source.len() > options.max_source_units { + check_length(source.encode_utf16().count(), options)?; + } + Ok(source.encode_utf16().collect()) +} + +fn check_length(units: usize, options: &Options) -> Result<(), SyntaxError> { + if units > options.max_source_units || u32::try_from(units).is_err() { + return Err(SyntaxError { message: TOO_LONG.into(), line: 1, column: 1, offset: 0 }); + } + Ok(()) +} + +/// Where each line of a source starts, for turning offsets into lines and columns. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct LineTable { + starts: Vec, + /// The source ends with a line terminator. + ends_with_break: bool, + empty: bool, +} + +impl LineTable { + pub(crate) fn new(src: &[u16]) -> Self { + let mut starts = vec![0]; + let mut i = 0; + while let Some(&u) = src.get(i) { + i += 1; + let terminator = match u { + 0x0D if src.get(i) == Some(&0x0A) => { + i += 1; + true + } + 0x0A | 0x0D | 0x2028 | 0x2029 => true, + _ => false, + }; + if terminator { + starts.push(u32::try_from(i).unwrap_or(u32::MAX)); + } + } + let ends_with_break = src.last().is_some_and(|&u| matches!(u, 0x0A | 0x0D | 0x2028 | 0x2029)); + Self { starts, ends_with_break, empty: src.is_empty() } + } + + pub(crate) fn line_column(&self, offset: u32) -> (u32, u32) { + let line = self.starts.partition_point(|&s| s <= offset).max(1); + let start = self.starts.get(line - 1).copied().unwrap_or(0); + (u32::try_from(line).unwrap_or(u32::MAX), offset.saturating_sub(start) + 1) + } + + pub(crate) fn line_count(&self) -> u32 { + if self.empty { + return 0; + } + let breaks = self.starts.len() - 1; + u32::try_from(breaks + usize::from(!self.ends_with_break)).unwrap_or(u32::MAX) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/js/src/number.rs b/crates/js/src/number.rs new file mode 100644 index 000000000..4afd48076 --- /dev/null +++ b/crates/js/src/number.rs @@ -0,0 +1,69 @@ +//! Number to string conversion (ECMA-262 5.1 section 9.8.1), for numeric property names. + +/// The string ToString gives for the number `x`: the shortest decimal digits that read back as +/// `x`, laid out as section 9.8.1 says. +pub fn to_string(x: f64) -> String { + if x.is_nan() { + return "NaN".into(); + } + if x == 0.0 { + return "0".into(); + } + if x.is_infinite() { + return if x > 0.0 { "Infinity".into() } else { "-Infinity".into() }; + } + if x < 0.0 { + return format!("-{}", to_string(-x)); + } + // Rust's `{:e}` writes the shortest digits that round-trip: `d.ddde±x`. + let sci = format!("{x:e}"); + let (mantissa, exp) = sci.split_once('e').unwrap_or((&sci, "0")); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + let k = digits.len() as i64; + let n = exp.parse::().unwrap_or(0) + 1; + if k <= n && n <= 21 { + let zeros = usize::try_from(n - k).unwrap_or(0); + format!("{digits}{}", "0".repeat(zeros)) + } else if 0 < n && n <= 21 { + let (int, frac) = digits.split_at_checked(usize::try_from(n).unwrap_or(0)).unwrap_or((&digits, "")); + format!("{int}.{frac}") + } else if -6 < n && n <= 0 { + let zeros = usize::try_from(-n).unwrap_or(0); + format!("0.{}{digits}", "0".repeat(zeros)) + } else { + let e = n - 1; + let sign = if e < 0 { '-' } else { '+' }; + let (first, rest) = digits.split_at_checked(1).unwrap_or((&digits, "")); + if rest.is_empty() { format!("{first}e{sign}{}", e.abs()) } else { format!("{first}.{rest}e{sign}{}", e.abs()) } + } +} + +#[cfg(test)] +mod tests { + use super::to_string; + + #[test] + fn numbers_print_as_section_9_8_1_lays_them_out() { + let cases: &[(f64, &str)] = &[ + (0.0, "0"), + (-0.0, "0"), + (1.0, "1"), + (-1.5, "-1.5"), + (123.456, "123.456"), + (1e21, "1e+21"), + (1e20, "100000000000000000000"), + (0.000001, "0.000001"), + (1e-7, "1e-7"), + (1.5e-7, "1.5e-7"), + (0.1 + 0.2, "0.30000000000000004"), + (f64::NAN, "NaN"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + ]; + for (x, want) in cases { + assert_eq!(to_string(*x), *want, "{x:?}"); + } + } +} diff --git a/crates/js/src/parser.rs b/crates/js/src/parser.rs new file mode 100644 index 000000000..d56b75617 --- /dev/null +++ b/crates/js/src/parser.rs @@ -0,0 +1,1236 @@ +//! The parser: tokens into a [`Program`] (ECMA-262 5.1 sections 11 to 14), with the early errors +//! of section 16 and of strict mode (Annex C). +//! +//! It is a recursive-descent parser whose recursion is counted: each statement, assignment +//! expression, primary expression and function body it enters adds one to a depth, and going past +//! [`crate::Options::max_depth`] gives a `SyntaxError`. Binary operators are read with an operator +//! stack, and prefix operators, assignment chains, `else if` chains and labels with loops, so long +//! runs of them do not nest at all. + +use std::collections::{HashMap, HashSet}; + +use crate::LineTable; +use crate::ast::{ + AssignOp, Ast, BinaryOp, CatchClause, Expr, ExprId, ForInTarget, ForInit, Function, FunctionKind, LogicalOp, Name, Program, Property, + PropertyKey, PropertyValue, Stmt, StmtId, SwitchCase, UnaryOp, UpdateOp, VarDecl, +}; +use crate::error::{SyntaxError, TOO_DEEP}; +use crate::lexer::{Goal, Lexer}; +use crate::token::{Directive, Keyword, Punct, Span, Token, TokenKind, strict_reserved}; + +type Result = std::result::Result; + +/// What the code being parsed allows, saved and replaced at each function body. +#[derive(Clone, Debug, Default)] +struct Context { + strict: bool, + in_function: bool, + /// The labels in scope, each with whether it labels an iteration statement (`continue` may + /// name only those). Labels in scope have distinct names. + labels: HashMap, + /// Iteration statements open in the current function. + loops: u32, + /// Iteration and `switch` statements open in the current function. + breakables: u32, +} + +/// A binary operator, with the logical ones apart. +#[derive(Clone, Copy, Debug)] +enum BinKind { + Binary(BinaryOp), + Logical(LogicalOp), +} + +/// A prefix operator. +#[derive(Clone, Copy, Debug)] +enum Prefix { + Unary(UnaryOp), + Update(UpdateOp), +} + +pub(crate) struct Parser<'s> { + src: &'s [u16], + lex: Lexer<'s>, + lines: LineTable, + tok: Token, + /// Where the last token read ends. + prev_end: u32, + ast: Ast, + depth: usize, + max_depth: usize, + ctx: Context, + directives: Vec, + /// The tokens read, kept for [`crate::tokenize`]. + tokens: Option>, + /// The next statement may be a function declaration outside strict mode code: it is an `if` + /// branch or the body of a label in a statement list (ECMA-262 2015 Annex B.3.2 and B.3.4). + /// Read and cleared as each statement starts. + function_allowed: bool, + /// The next statement stands in a statement list, where a label's body may be a function + /// declaration. Read and cleared as each statement starts. + in_list: bool, +} + +impl<'s> Parser<'s> { + pub(crate) fn new(src: &'s [u16], options: &crate::Options, keep_tokens: bool) -> Result { + let lex = Lexer::new(src, options.preprocessor_lines, options.max_depth); + let placeholder = + Token { kind: TokenKind::Eof, span: Span::default(), line: 1, column: 1, newline_before: false, escaped: false, legacy_octal: false }; + let mut parser = Self { + src, + lex, + lines: LineTable::new(src), + tok: placeholder, + prev_end: 0, + ast: Ast::default(), + depth: 0, + max_depth: options.max_depth, + ctx: Context { strict: options.strict, ..Context::default() }, + directives: Vec::new(), + tokens: keep_tokens.then(Vec::new), + function_allowed: false, + in_list: false, + }; + parser.tok = parser.next_significant()?; + Ok(parser) + } + + pub(crate) fn parse_program(mut self) -> Result<(Program, Vec)> { + let body = self.parse_body(false)?; + if self.tok.kind != TokenKind::Eof { + return Err(self.unexpected()); + } + if let Some(tokens) = &mut self.tokens { + tokens.push(self.tok.clone()); + } + let program = Program { ast: self.ast, body, strict: self.ctx.strict, directives: self.directives, lines: self.lines }; + Ok((program, self.tokens.unwrap_or_default())) + } + + // ---- tokens ---- + + /// The next token that is not a preprocessor line; preprocessor lines are kept. + fn next_significant(&mut self) -> Result { + loop { + let token = self.lex.next_token(Goal::Div)?; + if let TokenKind::Directive(directive) = &token.kind { + self.directives.push(directive.clone()); + if let Some(tokens) = &mut self.tokens { + tokens.push(token); + } + continue; + } + return Ok(token); + } + } + + /// Reads the current token and moves to the next. + fn bump(&mut self) -> Result { + // Kept before reading on, so a preprocessor line after the token comes after it. + if let Some(tokens) = &mut self.tokens { + tokens.push(self.tok.clone()); + } + let next = self.next_significant()?; + let token = std::mem::replace(&mut self.tok, next); + self.prev_end = token.span.end; + Ok(token) + } + + /// The token after the current one, without reading it. `None` when the lexer cannot read one + /// (reading it for real reports the error). + fn peek(&self) -> Option { + let mut lex = self.lex.clone(); + loop { + match lex.next_token(Goal::Div) { + Ok(Token { kind: TokenKind::Directive(_), .. }) => {} + Ok(token) => return Some(token), + Err(_) => return None, + } + } + } + + fn at(&self, p: Punct) -> bool { + self.tok.is_punct(p) + } + + fn at_keyword(&self, k: Keyword) -> bool { + self.tok.is_keyword(k) + } + + fn eat(&mut self, p: Punct) -> Result { + if self.at(p) { + self.bump()?; + return Ok(true); + } + Ok(false) + } + + fn expect(&mut self, p: Punct) -> Result { + if self.at(p) { + return self.bump(); + } + Err(self.error_at_token(&self.tok, format!("expected '{}' but found {}", p.as_str(), describe(&self.tok)))) + } + + fn expect_keyword(&mut self, k: Keyword) -> Result { + if self.at_keyword(k) { + return self.bump(); + } + Err(self.error_at_token(&self.tok, format!("expected '{}' but found {}", k.as_str(), describe(&self.tok)))) + } + + /// Ends a statement: a `;`, or one inserted before `}`, the end of the source or a token on a + /// new line (section 7.9). + fn semicolon(&mut self) -> Result<()> { + if self.eat(Punct::Semicolon)? || self.at(Punct::RBrace) || self.tok.kind == TokenKind::Eof || self.tok.newline_before { + return Ok(()); + } + Err(self.unexpected()) + } + + // ---- errors ---- + + fn error_at_offset(&self, offset: u32, message: impl Into) -> SyntaxError { + let (line, column) = self.lines.line_column(offset); + SyntaxError { message: message.into(), line, column, offset } + } + + fn error_at_token(&self, token: &Token, message: impl Into) -> SyntaxError { + SyntaxError { message: message.into(), line: token.line, column: token.column, offset: token.span.start } + } + + fn unexpected(&self) -> SyntaxError { + self.error_at_token(&self.tok, format!("unexpected {}", describe(&self.tok))) + } + + /// Counts one more level of nesting; past the limit, a `SyntaxError` at the current token. + fn enter(&mut self) -> Result<()> { + if self.depth >= self.max_depth { + return Err(self.error_at_token(&self.tok, TOO_DEEP)); + } + self.depth += 1; + Ok(()) + } + + fn leave(&mut self) { + self.depth = self.depth.saturating_sub(1); + } + + fn span_from(&self, start: Span) -> Span { + Span::new(start.start, self.prev_end.max(start.start)) + } + + // ---- names ---- + + /// Checks a name used as an identifier: not a reserved word written with escapes, and not a + /// strict-mode reserved word in strict code. + fn check_identifier(&self, token: &Token, name: &str) -> Result<()> { + if token.escaped && Keyword::from_word(name).is_some() { + return Err(self.error_at_token(token, format!("keyword '{name}' must not contain escaped characters"))); + } + if self.ctx.strict && strict_reserved(name) { + return Err(self.error_at_token(token, format!("'{name}' is a reserved word in strict mode"))); + } + Ok(()) + } + + /// Reads an identifier that a declaration binds: checked as [`Self::check_identifier`] does, + /// and in strict code neither `eval` nor `arguments`. + fn binding_identifier(&mut self) -> Result { + let TokenKind::Ident(name) = &self.tok.kind else { + return Err(self.error_at_token(&self.tok, format!("expected an identifier but found {}", describe(&self.tok)))); + }; + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + if self.ctx.strict && is_eval_or_arguments(&name) { + return Err(self.error_at_token(&self.tok, format!("'{name}' cannot be declared in strict mode"))); + } + let token = self.bump()?; + Ok(Name { name, span: token.span }) + } + + // ---- program and function bodies ---- + + /// Reads source elements up to `}` (in a function body) or the end of the source, reading the + /// directive prologue at their start (section 14.1). A `"use strict"` directive makes the rest + /// strict mode code. + fn parse_body(&mut self, in_function: bool) -> Result> { + let mut body = Vec::new(); + let mut prologue = true; + // A string with an octal escape before a `"use strict"` directive in the same prologue. + let mut octal: Option = None; + while !(self.tok.kind == TokenKind::Eof || (in_function && self.at(Punct::RBrace))) { + if prologue && matches!(self.tok.kind, TokenKind::String(_)) { + let first = self.tok.clone(); + let stmt = self.parse_statement_item()?; + body.push(stmt); + if !self.is_directive(stmt, &first) { + prologue = false; + continue; + } + if self.is_use_strict(&first) { + self.ctx.strict = true; + if let Some(octal) = &octal { + return Err(self.error_at_token(octal, "octal escape sequences are not allowed in strict mode")); + } + } + if first.legacy_octal && octal.is_none() { + octal = Some(first); + } + continue; + } + prologue = false; + body.push(self.parse_statement_item()?); + } + Ok(body) + } + + /// Whether `stmt`, which started at `first`, is a directive: an expression statement made of + /// that one string literal. + fn is_directive(&self, stmt: StmtId, first: &Token) -> bool { + match self.ast.stmt(stmt) { + Some(Stmt::Expr(e)) => matches!(self.ast.expr(*e), Some(Expr::String(_))) && self.ast.expr_span(*e) == first.span, + _ => false, + } + } + + /// The string token is exactly `"use strict"` or `'use strict'`, without escapes or line + /// continuations. + fn is_use_strict(&self, token: &Token) -> bool { + let raw = self.src.get(token.span.start as usize..token.span.end as usize).unwrap_or_default(); + raw.len() == 12 && raw.get(1..11).is_some_and(|inner| inner.iter().copied().eq("use strict".encode_utf16())) + } + + // ---- statements ---- + + /// A source element: a function declaration or a statement. + fn parse_statement_item(&mut self) -> Result { + if self.at_keyword(Keyword::Function) { + self.enter()?; + let result = self.parse_function_declaration(); + self.leave(); + return result; + } + self.in_list = true; + self.parse_statement() + } + + fn parse_statement(&mut self) -> Result { + self.enter()?; + let result = self.statement(); + self.leave(); + result + } + + /// A statement where Annex B lets sloppy code put a function declaration: an `if` branch. + fn parse_branch(&mut self) -> Result { + self.function_allowed = !self.ctx.strict; + self.parse_statement() + } + + fn statement(&mut self) -> Result { + let start = self.tok.span; + let function_allowed = std::mem::take(&mut self.function_allowed); + let in_list = std::mem::take(&mut self.in_list); + match &self.tok.kind { + TokenKind::Keyword(Keyword::Function) => { + if !function_allowed { + let message = if self.ctx.strict { + "in strict mode code, functions can only be declared at top level or inside a block" + } else { + "a function declaration cannot stand here; put it in a block" + }; + return Err(self.error_at_token(&self.tok, message)); + } + self.parse_function_declaration() + } + TokenKind::Ident(_) if self.peek().is_some_and(|t| t.is_punct(Punct::Colon)) => self.parse_labeled(in_list), + _ => self.plain_statement(start), + } + } + + fn plain_statement(&mut self, start: Span) -> Result { + match &self.tok.kind { + TokenKind::Punct(Punct::LBrace) => { + let body = self.parse_block()?; + Ok(self.ast.push_stmt(Stmt::Block(body), self.span_from(start))) + } + TokenKind::Punct(Punct::Semicolon) => { + self.bump()?; + Ok(self.ast.push_stmt(Stmt::Empty, start)) + } + TokenKind::Keyword(k) => self.keyword_statement(*k, start), + _ => { + let expr = self.parse_expression(false)?; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Expr(expr), self.span_from(start))) + } + } + } + + fn keyword_statement(&mut self, k: Keyword, start: Span) -> Result { + match k { + Keyword::Var => { + self.bump()?; + let decls = self.parse_var_decls(false)?; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Var(decls), self.span_from(start))) + } + Keyword::If => self.parse_if(), + Keyword::Do => self.parse_do_while(start), + Keyword::While => self.parse_while(start), + Keyword::For => self.parse_for(start), + Keyword::Continue | Keyword::Break => self.parse_jump(k == Keyword::Break, start), + Keyword::Return => self.parse_return(start), + Keyword::With => self.parse_with(start), + Keyword::Switch => self.parse_switch(start), + Keyword::Throw => self.parse_throw(start), + Keyword::Try => self.parse_try(start), + Keyword::Debugger => { + self.bump()?; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Debugger, self.span_from(start))) + } + _ => { + let expr = self.parse_expression(false)?; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Expr(expr), self.span_from(start))) + } + } + } + + /// `{ … }`: its statements. + fn parse_block(&mut self) -> Result> { + self.expect(Punct::LBrace)?; + let mut body = Vec::new(); + while !self.at(Punct::RBrace) { + if self.tok.kind == TokenKind::Eof { + return Err(self.unexpected()); + } + body.push(self.parse_statement_item()?); + } + self.bump()?; + Ok(body) + } + + /// `var` declarations after the `var`. `no_in` leaves `in` to a `for (… in …)` around them. + fn parse_var_decls(&mut self, no_in: bool) -> Result> { + let mut decls = Vec::new(); + loop { + let name = self.binding_identifier()?; + let init = if self.eat(Punct::Eq)? { Some(self.parse_assignment(no_in)?) } else { None }; + decls.push(VarDecl { name, init }); + if !self.eat(Punct::Comma)? { + return Ok(decls); + } + } + } + + /// `if (…) … else if (…) … else …`, the `else if` chain read with a loop. + fn parse_if(&mut self) -> Result { + let mut chain = Vec::new(); + let alternate = loop { + let start = self.expect_keyword(Keyword::If)?.span; + self.expect(Punct::LParen)?; + let test = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + let consequent = self.parse_branch()?; + chain.push((start, test, consequent)); + if !self.at_keyword(Keyword::Else) { + break None; + } + self.bump()?; + if !self.at_keyword(Keyword::If) { + break Some(self.parse_branch()?); + } + }; + let mut alternate = alternate; + let mut last = None; + for (start, test, consequent) in chain.into_iter().rev() { + let id = self.ast.push_stmt(Stmt::If { test, consequent, alternate }, self.span_from(start)); + alternate = Some(id); + last = Some(id); + } + last.ok_or_else(|| self.unexpected()) + } + + /// The body of an iteration statement, with `break` and `continue` allowed in it. + fn parse_loop_body(&mut self) -> Result { + self.ctx.loops += 1; + self.ctx.breakables += 1; + let body = self.parse_statement(); + self.ctx.loops -= 1; + self.ctx.breakables -= 1; + body + } + + fn parse_do_while(&mut self, start: Span) -> Result { + self.bump()?; + let body = self.parse_loop_body()?; + self.expect_keyword(Keyword::While)?; + self.expect(Punct::LParen)?; + let test = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + // A `;` after `do … while (…)` may be left out even on the same line, as ECMA-262 2015 + // section 11.9.1 allows. + self.eat(Punct::Semicolon)?; + Ok(self.ast.push_stmt(Stmt::DoWhile { body, test }, self.span_from(start))) + } + + fn parse_while(&mut self, start: Span) -> Result { + self.bump()?; + self.expect(Punct::LParen)?; + let test = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + let body = self.parse_loop_body()?; + Ok(self.ast.push_stmt(Stmt::While { test, body }, self.span_from(start))) + } + + fn parse_for(&mut self, start: Span) -> Result { + self.bump()?; + self.expect(Punct::LParen)?; + let init = if self.at(Punct::Semicolon) { + None + } else if self.at_keyword(Keyword::Var) { + self.bump()?; + let mut decls = self.parse_var_decls(true)?; + if decls.len() == 1 && self.at_keyword(Keyword::In) { + let decl = decls.pop().ok_or_else(|| self.unexpected())?; + // The variable of a for-in loop takes an initializer only outside strict mode + // code (ECMA-262 2017 Annex B.3.6). + if self.ctx.strict && decl.init.is_some() { + return Err(self.error_at_token(&self.tok, "a for-in variable cannot have an initializer in strict mode code")); + } + return self.parse_for_in(ForInTarget::Var(decl), start); + } + Some(ForInit::Var(decls)) + } else { + let expr = self.parse_expression(true)?; + if self.at_keyword(Keyword::In) { + self.check_target(expr, "invalid left-hand side in for-in loop")?; + return self.parse_for_in(ForInTarget::Expr(expr), start); + } + Some(ForInit::Expr(expr)) + }; + self.expect(Punct::Semicolon)?; + let test = if self.at(Punct::Semicolon) { None } else { Some(self.parse_expression(false)?) }; + self.expect(Punct::Semicolon)?; + let update = if self.at(Punct::RParen) { None } else { Some(self.parse_expression(false)?) }; + self.expect(Punct::RParen)?; + let body = self.parse_loop_body()?; + Ok(self.ast.push_stmt(Stmt::For { init, test, update, body }, self.span_from(start))) + } + + /// The rest of `for (target in object) body`, from `in`. + fn parse_for_in(&mut self, target: ForInTarget, start: Span) -> Result { + self.expect_keyword(Keyword::In)?; + let object = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + let body = self.parse_loop_body()?; + Ok(self.ast.push_stmt(Stmt::ForIn { target, object, body }, self.span_from(start))) + } + + /// `break` or `continue`, with an optional label on the same line, which must name a label + /// around it in the same function (an iteration statement's, for `continue`). + fn parse_jump(&mut self, is_break: bool, start: Span) -> Result { + let keyword = self.bump()?; + let label = match &self.tok.kind { + TokenKind::Ident(name) if !self.tok.newline_before => { + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + let token = self.bump()?; + match self.ctx.labels.get(&name) { + None => return Err(self.error_at_token(&token, format!("undefined label '{name}'"))), + Some(&is_loop) if !is_break && !is_loop => { + return Err(self.error_at_token(&token, format!("label '{name}' does not name a loop, so continue cannot use it"))); + } + Some(_) => {} + } + Some(Name { name, span: token.span }) + } + _ => None, + }; + if label.is_none() { + if is_break && self.ctx.breakables == 0 { + return Err(self.error_at_token(&keyword, "break outside a loop or switch")); + } + if !is_break && self.ctx.loops == 0 { + return Err(self.error_at_token(&keyword, "continue outside a loop")); + } + } + self.semicolon()?; + let stmt = if is_break { Stmt::Break(label) } else { Stmt::Continue(label) }; + Ok(self.ast.push_stmt(stmt, self.span_from(start))) + } + + fn parse_return(&mut self, start: Span) -> Result { + let keyword = self.bump()?; + if !self.ctx.in_function { + return Err(self.error_at_token(&keyword, "return outside a function")); + } + let ends = self.at(Punct::Semicolon) || self.at(Punct::RBrace) || self.tok.kind == TokenKind::Eof || self.tok.newline_before; + let arg = if ends { None } else { Some(self.parse_expression(false)?) }; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Return(arg), self.span_from(start))) + } + + fn parse_with(&mut self, start: Span) -> Result { + if self.ctx.strict { + return Err(self.error_at_token(&self.tok, "with statements are not allowed in strict mode")); + } + self.bump()?; + self.expect(Punct::LParen)?; + let object = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + let body = self.parse_statement()?; + Ok(self.ast.push_stmt(Stmt::With { object, body }, self.span_from(start))) + } + + fn parse_switch(&mut self, start: Span) -> Result { + self.bump()?; + self.expect(Punct::LParen)?; + let discriminant = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + self.expect(Punct::LBrace)?; + self.ctx.breakables += 1; + let cases = self.parse_cases(); + self.ctx.breakables -= 1; + let cases = cases?; + Ok(self.ast.push_stmt(Stmt::Switch { discriminant, cases }, self.span_from(start))) + } + + /// A `switch` statement's clauses, up to and with its `}`; at most one `default`. + fn parse_cases(&mut self) -> Result> { + let mut cases = Vec::new(); + let mut has_default = false; + while !self.eat(Punct::RBrace)? { + let case_start = self.tok.span; + let test = if self.at_keyword(Keyword::Case) { + self.bump()?; + Some(self.parse_expression(false)?) + } else if self.at_keyword(Keyword::Default) { + if has_default { + return Err(self.error_at_token(&self.tok, "more than one default clause in switch statement")); + } + has_default = true; + self.bump()?; + None + } else { + return Err(self.unexpected()); + }; + self.expect(Punct::Colon)?; + let mut body = Vec::new(); + while !(self.at_keyword(Keyword::Case) || self.at_keyword(Keyword::Default) || self.at(Punct::RBrace)) { + if self.tok.kind == TokenKind::Eof { + return Err(self.unexpected()); + } + body.push(self.parse_statement_item()?); + } + cases.push(SwitchCase { test, body, span: self.span_from(case_start) }); + } + Ok(cases) + } + + fn parse_throw(&mut self, start: Span) -> Result { + self.bump()?; + if self.tok.newline_before { + return Err(self.error_at_token(&self.tok, "line break after throw")); + } + let arg = self.parse_expression(false)?; + self.semicolon()?; + Ok(self.ast.push_stmt(Stmt::Throw(arg), self.span_from(start))) + } + + fn parse_try(&mut self, start: Span) -> Result { + self.bump()?; + let block = self.parse_block()?; + let handler = if self.at_keyword(Keyword::Catch) { + self.bump()?; + self.expect(Punct::LParen)?; + let param = self.binding_identifier()?; + self.expect(Punct::RParen)?; + Some(CatchClause { param, body: self.parse_block()? }) + } else { + None + }; + let finalizer = if self.at_keyword(Keyword::Finally) { + self.bump()?; + Some(self.parse_block()?) + } else { + None + }; + if handler.is_none() && finalizer.is_none() { + return Err(self.error_at_token(&self.tok, "missing catch or finally after try")); + } + Ok(self.ast.push_stmt(Stmt::Try { block, handler, finalizer }, self.span_from(start))) + } + + /// `a: b: statement`: the labels read with a loop, each unique among the labels around it. + /// `in_list`: the labeled statement stands in a statement list, so outside strict mode code + /// its body may be a function declaration. + fn parse_labeled(&mut self, in_list: bool) -> Result { + let mut labels: Vec<(Name, Span)> = Vec::new(); + let mut chain = HashSet::new(); + while let TokenKind::Ident(name) = &self.tok.kind { + if !self.peek().is_some_and(|t| t.is_punct(Punct::Colon)) { + break; + } + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + if self.ctx.labels.contains_key(&name) || !chain.insert(name.clone()) { + return Err(self.error_at_token(&self.tok, format!("duplicate label '{name}'"))); + } + let token = self.bump()?; + self.bump()?; + labels.push((Name { name, span: token.span }, token.span)); + } + let is_loop = self.at_keyword(Keyword::For) || self.at_keyword(Keyword::While) || self.at_keyword(Keyword::Do); + self.ctx.labels.extend(chain.into_iter().map(|name| (name, is_loop))); + self.function_allowed = in_list && !self.ctx.strict; + let body = self.parse_statement(); + for (label, _) in &labels { + self.ctx.labels.remove(&label.name); + } + let mut body = body?; + for (label, label_start) in labels.into_iter().rev() { + let span = Span::new(label_start.start, self.ast.stmt_span(body).end); + body = self.ast.push_stmt(Stmt::Labeled { label, body }, span); + } + Ok(body) + } + + // ---- functions ---- + + fn parse_function_declaration(&mut self) -> Result { + let start = self.expect_keyword(Keyword::Function)?.span; + let name = self.function_name()?; + let id = self.parse_function_rest(FunctionKind::Declaration, Some(name), start)?; + Ok(self.ast.push_stmt(Stmt::FunctionDecl(id), self.span_from(start))) + } + + /// A function's name, checked as an identifier; the strict-mode checks wait for the body, + /// whose prologue may make the function strict. + fn function_name(&mut self) -> Result<(Name, Token)> { + let TokenKind::Ident(name) = &self.tok.kind else { + return Err(self.error_at_token(&self.tok, format!("expected a function name but found {}", describe(&self.tok)))); + }; + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + let token = self.bump()?; + Ok((Name { name, span: token.span }, token)) + } + + /// The parameters and body of a function, from `(`. + fn parse_function_rest(&mut self, kind: FunctionKind, name: Option<(Name, Token)>, start: Span) -> Result { + self.enter()?; + let result = self.function_rest(kind, name, start); + self.leave(); + result + } + + fn function_rest(&mut self, kind: FunctionKind, name: Option<(Name, Token)>, start: Span) -> Result { + self.expect(Punct::LParen)?; + let mut params: Vec<(Name, Token)> = Vec::new(); + if !self.at(Punct::RParen) { + loop { + let (param, token) = self.function_name_or_param()?; + params.push((param, token)); + if !self.eat(Punct::Comma)? { + break; + } + } + } + let close = self.expect(Punct::RParen)?; + match kind { + FunctionKind::Getter if !params.is_empty() => return Err(self.error_at_token(&close, "a getter must not have parameters")), + FunctionKind::Setter if params.len() != 1 => return Err(self.error_at_token(&close, "a setter must have exactly one parameter")), + _ => {} + } + self.expect(Punct::LBrace)?; + let inner = Context { strict: self.ctx.strict, in_function: true, ..Context::default() }; + let outer = std::mem::replace(&mut self.ctx, inner); + let body = self.parse_body(true); + let strict = self.ctx.strict; + self.ctx = outer; + let body = body?; + self.expect(Punct::RBrace)?; + if strict { + self.check_strict_function(name.as_ref(), ¶ms)?; + } + Ok(self.ast.push_function(Function { + kind, + name: name.map(|(n, _)| n), + params: params.into_iter().map(|(n, _)| n).collect(), + body, + strict, + span: self.span_from(start), + })) + } + + /// A parameter's name. + fn function_name_or_param(&mut self) -> Result<(Name, Token)> { + let TokenKind::Ident(name) = &self.tok.kind else { + return Err(self.error_at_token(&self.tok, format!("expected a parameter name but found {}", describe(&self.tok)))); + }; + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + let token = self.bump()?; + Ok((Name { name, span: token.span }, token)) + } + + /// The rules strict mode code sets for a function's name and parameters (Annex C): neither + /// `eval`, `arguments` nor a strict-mode reserved word, and no parameter name twice. + fn check_strict_function(&self, name: Option<&(Name, Token)>, params: &[(Name, Token)]) -> Result<()> { + for (n, token) in name.into_iter().chain(params) { + if is_eval_or_arguments(&n.name) { + return Err(self.error_at_token(token, format!("'{}' cannot be declared in strict mode", n.name))); + } + if strict_reserved(&n.name) { + return Err(self.error_at_token(token, format!("'{}' is a reserved word in strict mode", n.name))); + } + } + let mut seen = HashSet::new(); + for (p, token) in params { + if !seen.insert(p.name.as_str()) { + return Err(self.error_at_token(token, format!("duplicate parameter name '{}' is not allowed in strict mode", p.name))); + } + } + Ok(()) + } + + // ---- expressions ---- + + /// An expression: assignment expressions separated by commas. `no_in` leaves `in` to a + /// `for (… in …)` around it. + fn parse_expression(&mut self, no_in: bool) -> Result { + let first = self.parse_assignment(no_in)?; + if !self.at(Punct::Comma) { + return Ok(first); + } + let mut list = vec![first]; + while self.eat(Punct::Comma)? { + list.push(self.parse_assignment(no_in)?); + } + let span = self.ast.expr_span(first).to(Span::new(self.prev_end, self.prev_end)); + Ok(self.ast.push_expr(Expr::Sequence(list), span)) + } + + fn parse_assignment(&mut self, no_in: bool) -> Result { + self.enter()?; + let result = self.assignment(no_in); + self.leave(); + result + } + + /// `a = b += c`: the targets kept on a stack, so a long chain does not nest. + fn assignment(&mut self, no_in: bool) -> Result { + let mut targets: Vec<(ExprId, AssignOp)> = Vec::new(); + loop { + let expr = self.parse_conditional(no_in)?; + let Some(op) = assign_op(&self.tok) else { + let mut value = expr; + while let Some((target, op)) = targets.pop() { + let span = self.ast.expr_span(target).to(self.ast.expr_span(value)); + value = self.ast.push_expr(Expr::Assign { op, target, value }, span); + } + return Ok(value); + }; + self.check_target(expr, "invalid assignment target")?; + self.bump()?; + targets.push((expr, op)); + } + } + + /// An assignment's, `++`'s, `--`'s or `for-in`'s target: an identifier (in strict code + /// neither `eval` nor `arguments`), a property, or outside strict mode code a call (assigning + /// to its result fails at run time, as ECMA-262 2015 Annex B and web browsers have it). + fn check_target(&self, expr: ExprId, message: &str) -> Result<()> { + let inner = self.ast.unparenthesized(expr); + match self.ast.expr(inner) { + Some(Expr::Ident(name)) => { + if self.ctx.strict && is_eval_or_arguments(name) { + return Err(self.error_at_offset(self.ast.expr_span(inner).start, format!("cannot assign to '{name}' in strict mode"))); + } + Ok(()) + } + Some(Expr::Member { .. } | Expr::Index { .. }) => Ok(()), + Some(Expr::Call { .. }) if !self.ctx.strict => Ok(()), + _ => Err(self.error_at_offset(self.ast.expr_span(expr).start, message)), + } + } + + fn parse_conditional(&mut self, no_in: bool) -> Result { + let test = self.parse_binary(no_in)?; + if !self.at(Punct::Question) { + return Ok(test); + } + self.bump()?; + let consequent = self.parse_assignment(false)?; + self.expect(Punct::Colon)?; + let alternate = self.parse_assignment(no_in)?; + let span = self.ast.expr_span(test).to(self.ast.expr_span(alternate)); + Ok(self.ast.push_expr(Expr::Conditional { test, consequent, alternate }, span)) + } + + /// Binary and logical operators, read with an operator stack by precedence (all are left + /// associative). + fn parse_binary(&mut self, no_in: bool) -> Result { + let mut operands = vec![self.parse_unary()?]; + let mut ops: Vec<(BinKind, u8)> = Vec::new(); + while let Some((op, precedence)) = binary_op(&self.tok, no_in) { + while ops.last().is_some_and(|&(_, top)| top >= precedence) { + self.reduce(&mut operands, &mut ops); + } + self.bump()?; + ops.push((op, precedence)); + operands.push(self.parse_unary()?); + } + while !ops.is_empty() { + self.reduce(&mut operands, &mut ops); + } + operands.pop().ok_or_else(|| self.unexpected()) + } + + /// Joins the top two operands with the top operator. + fn reduce(&mut self, operands: &mut Vec, ops: &mut Vec<(BinKind, u8)>) { + let (Some((op, _)), Some(right), Some(left)) = (ops.pop(), operands.pop(), operands.pop()) else { + return; + }; + let span = self.ast.expr_span(left).to(self.ast.expr_span(right)); + let expr = match op { + BinKind::Binary(op) => Expr::Binary { op, left, right }, + BinKind::Logical(op) => Expr::Logical { op, left, right }, + }; + operands.push(self.ast.push_expr(expr, span)); + } + + /// Prefix operators, read with a loop and applied innermost first. + fn parse_unary(&mut self) -> Result { + let mut prefixes = Vec::new(); + while let Some(prefix) = prefix_op(&self.tok) { + prefixes.push((prefix, self.tok.span)); + self.bump()?; + } + let mut expr = self.parse_postfix()?; + for (prefix, start) in prefixes.into_iter().rev() { + let span = start.to(self.ast.expr_span(expr)); + expr = match prefix { + Prefix::Update(op) => { + self.check_target(expr, "invalid increment or decrement operand")?; + self.ast.push_expr(Expr::Update { op, prefix: true, target: expr }, span) + } + Prefix::Unary(op) => { + if op == UnaryOp::Delete && self.ctx.strict && matches!(self.ast.expr(self.ast.unparenthesized(expr)), Some(Expr::Ident(_))) { + return Err(self.error_at_offset(start.start, "delete of an unqualified identifier in strict mode")); + } + self.ast.push_expr(Expr::Unary { op, arg: expr }, span) + } + }; + } + Ok(expr) + } + + fn parse_postfix(&mut self) -> Result { + let expr = self.parse_left_hand_side()?; + let op = match &self.tok.kind { + TokenKind::Punct(Punct::PlusPlus) if !self.tok.newline_before => UpdateOp::Increment, + TokenKind::Punct(Punct::MinusMinus) if !self.tok.newline_before => UpdateOp::Decrement, + _ => return Ok(expr), + }; + self.check_target(expr, "invalid increment or decrement operand")?; + let token = self.bump()?; + let span = self.ast.expr_span(expr).to(token.span); + Ok(self.ast.push_expr(Expr::Update { op, prefix: false, target: expr }, span)) + } + + /// Member accesses, calls and `new` (section 11.2): each `new` takes the first argument list + /// after its callee, and a `new` left without one constructs with no arguments. + fn parse_left_hand_side(&mut self) -> Result { + let mut news = Vec::new(); + while self.at_keyword(Keyword::New) { + news.push(self.bump()?.span); + } + let mut expr = self.parse_primary()?; + loop { + match &self.tok.kind { + TokenKind::Punct(Punct::Dot) => { + self.bump()?; + let Some(name) = self.tok.identifier_name().map(str::to_string) else { + return Err(self.error_at_token(&self.tok, format!("expected a property name but found {}", describe(&self.tok)))); + }; + let token = self.bump()?; + let span = self.ast.expr_span(expr).to(token.span); + expr = self.ast.push_expr(Expr::Member { object: expr, property: Name { name, span: token.span } }, span); + } + TokenKind::Punct(Punct::LBracket) => { + self.bump()?; + let index = self.parse_expression(false)?; + let close = self.expect(Punct::RBracket)?; + let span = self.ast.expr_span(expr).to(close.span); + expr = self.ast.push_expr(Expr::Index { object: expr, index }, span); + } + TokenKind::Punct(Punct::LParen) => { + let args = self.parse_arguments()?; + let span = Span::new(self.ast.expr_span(expr).start, self.prev_end); + expr = match news.pop() { + Some(new) => self.ast.push_expr(Expr::New { callee: expr, args }, Span::new(new.start, self.prev_end)), + None => self.ast.push_expr(Expr::Call { callee: expr, args }, span), + }; + } + _ => break, + } + } + while let Some(new) = news.pop() { + let span = Span::new(new.start, self.ast.expr_span(expr).end); + expr = self.ast.push_expr(Expr::New { callee: expr, args: Vec::new() }, span); + } + Ok(expr) + } + + fn parse_arguments(&mut self) -> Result> { + self.expect(Punct::LParen)?; + let mut args = Vec::new(); + if !self.at(Punct::RParen) { + loop { + args.push(self.parse_assignment(false)?); + if !self.eat(Punct::Comma)? { + break; + } + } + } + self.expect(Punct::RParen)?; + Ok(args) + } + + fn parse_primary(&mut self) -> Result { + self.enter()?; + let result = self.primary(); + self.leave(); + result + } + + fn primary(&mut self) -> Result { + let start = self.tok.span; + let expr = match &self.tok.kind { + TokenKind::Keyword(Keyword::This) => Expr::This, + TokenKind::Keyword(Keyword::Null) => Expr::Null, + TokenKind::Keyword(Keyword::True) => Expr::Bool(true), + TokenKind::Keyword(Keyword::False) => Expr::Bool(false), + TokenKind::Keyword(Keyword::Function) => return self.parse_function_expression(), + TokenKind::Ident(name) => { + let name = name.clone(); + self.check_identifier(&self.tok, &name)?; + Expr::Ident(name) + } + TokenKind::Number(n) => { + if self.ctx.strict && self.tok.legacy_octal { + return Err(self.error_at_token(&self.tok, "octal literals are not allowed in strict mode")); + } + Expr::Number(*n) + } + TokenKind::String(s) => { + if self.ctx.strict && self.tok.legacy_octal { + return Err(self.error_at_token(&self.tok, "octal escape sequences are not allowed in strict mode")); + } + Expr::String(s.clone()) + } + TokenKind::Punct(Punct::Slash | Punct::SlashEq) => { + let regexp = self.lex.rescan_regexp(&self.tok)?; + self.tok = regexp; + let TokenKind::RegExp { pattern, flags } = &self.tok.kind else { + return Err(self.unexpected()); + }; + Expr::RegExp { pattern: pattern.clone(), flags: flags.clone() } + } + TokenKind::Punct(Punct::LBracket) => return self.parse_array(), + TokenKind::Punct(Punct::LBrace) => return self.parse_object(), + TokenKind::Punct(Punct::LParen) => { + self.bump()?; + let inner = self.parse_expression(false)?; + self.expect(Punct::RParen)?; + return Ok(self.ast.push_expr(Expr::Paren(inner), self.span_from(start))); + } + _ => return Err(self.unexpected()), + }; + let token = self.bump()?; + Ok(self.ast.push_expr(expr, token.span)) + } + + fn parse_function_expression(&mut self) -> Result { + let start = self.expect_keyword(Keyword::Function)?.span; + let name = if matches!(self.tok.kind, TokenKind::Ident(_)) { Some(self.function_name()?) } else { None }; + let id = self.parse_function_rest(FunctionKind::Expression, name, start)?; + Ok(self.ast.push_expr(Expr::Function(id), self.span_from(start))) + } + + /// `[a, , b]`: elements, a comma with nothing before it leaving a hole. + fn parse_array(&mut self) -> Result { + let start = self.expect(Punct::LBracket)?.span; + let mut elements = Vec::new(); + loop { + if self.eat(Punct::RBracket)? { + break; + } + if self.eat(Punct::Comma)? { + elements.push(None); + continue; + } + elements.push(Some(self.parse_assignment(false)?)); + if !self.at(Punct::RBracket) { + self.expect(Punct::Comma)?; + } + } + Ok(self.ast.push_expr(Expr::Array(elements), self.span_from(start))) + } + + /// `{a: 1, "b": 2, 3: c, get d() {…}, set d(v) {…}}`. A name may repeat, a later definition + /// replacing an earlier one (ECMA-262 2015 section 12.2.6.1). + fn parse_object(&mut self) -> Result { + let start = self.expect(Punct::LBrace)?.span; + let mut properties = Vec::new(); + while !self.eat(Punct::RBrace)? { + let property = self.parse_property()?; + properties.push(property); + if !self.at(Punct::RBrace) { + self.expect(Punct::Comma)?; + } + } + Ok(self.ast.push_expr(Expr::Object(properties), self.span_from(start))) + } + + fn parse_property(&mut self) -> Result { + let start = self.tok.span; + // `get` and `set` start an accessor only when written without escapes. + let accessor = match self.tok.ident().filter(|_| !self.tok.escaped) { + Some("get") => Some(FunctionKind::Getter), + Some("set") => Some(FunctionKind::Setter), + _ => None, + }; + if let Some(kind) = accessor { + let word = self.bump()?; + if property_key(&self.tok).is_some() { + let (key, _) = self.parse_property_key()?; + let function = self.parse_function_rest(kind, None, start)?; + let value = if kind == FunctionKind::Getter { PropertyValue::Get(function) } else { PropertyValue::Set(function) }; + return Ok(Property { key, value, span: self.span_from(start) }); + } + let key = PropertyKey::Ident(word.ident().unwrap_or_default().to_string()); + return self.property_value(key, start); + } + let (key, _) = self.parse_property_key()?; + self.property_value(key, start) + } + + /// `: value` after a data property's name. + fn property_value(&mut self, key: PropertyKey, start: Span) -> Result { + self.expect(Punct::Colon)?; + let value = self.parse_assignment(false)?; + Ok(Property { key, value: PropertyValue::Init(value), span: self.span_from(start) }) + } + + fn parse_property_key(&mut self) -> Result<(PropertyKey, Token)> { + let Some(key) = property_key(&self.tok) else { + return Err(self.error_at_token(&self.tok, format!("expected a property name but found {}", describe(&self.tok)))); + }; + if self.ctx.strict && self.tok.legacy_octal { + return Err(self.error_at_token(&self.tok, "octal literals and escape sequences are not allowed in strict mode")); + } + let token = self.bump()?; + Ok((key, token)) + } +} + +/// The key a token gives as a property name: an identifier name, a string or a number. +fn property_key(token: &Token) -> Option { + match &token.kind { + TokenKind::String(s) => Some(PropertyKey::String(s.clone())), + TokenKind::Number(n) => Some(PropertyKey::Number(*n)), + _ => token.identifier_name().map(|n| PropertyKey::Ident(n.to_string())), + } +} + +fn is_eval_or_arguments(name: &str) -> bool { + name == "eval" || name == "arguments" +} + +fn assign_op(token: &Token) -> Option { + let TokenKind::Punct(p) = &token.kind else { return None }; + Some(match p { + Punct::Eq => None, + Punct::PlusEq => Some(BinaryOp::Add), + Punct::MinusEq => Some(BinaryOp::Sub), + Punct::StarEq => Some(BinaryOp::Mul), + Punct::SlashEq => Some(BinaryOp::Div), + Punct::PercentEq => Some(BinaryOp::Mod), + Punct::ShlEq => Some(BinaryOp::Shl), + Punct::ShrEq => Some(BinaryOp::Shr), + Punct::UShrEq => Some(BinaryOp::UShr), + Punct::AmpEq => Some(BinaryOp::BitAnd), + Punct::PipeEq => Some(BinaryOp::BitOr), + Punct::CaretEq => Some(BinaryOp::BitXor), + _ => return None, + }) +} + +/// A binary operator and its precedence (higher binds tighter); `in` is left out under `no_in`. +fn binary_op(token: &Token, no_in: bool) -> Option<(BinKind, u8)> { + use BinKind::{Binary as B, Logical as L}; + match &token.kind { + TokenKind::Keyword(Keyword::Instanceof) => Some((B(BinaryOp::Instanceof), 7)), + TokenKind::Keyword(Keyword::In) if !no_in => Some((B(BinaryOp::In), 7)), + TokenKind::Punct(p) => Some(match p { + Punct::PipePipe => (L(LogicalOp::Or), 1), + Punct::AmpAmp => (L(LogicalOp::And), 2), + Punct::Pipe => (B(BinaryOp::BitOr), 3), + Punct::Caret => (B(BinaryOp::BitXor), 4), + Punct::Amp => (B(BinaryOp::BitAnd), 5), + Punct::EqEq => (B(BinaryOp::Eq), 6), + Punct::NotEq => (B(BinaryOp::NotEq), 6), + Punct::EqEqEq => (B(BinaryOp::StrictEq), 6), + Punct::NotEqEq => (B(BinaryOp::StrictNotEq), 6), + Punct::Lt => (B(BinaryOp::Lt), 7), + Punct::Gt => (B(BinaryOp::Gt), 7), + Punct::LtEq => (B(BinaryOp::LtEq), 7), + Punct::GtEq => (B(BinaryOp::GtEq), 7), + Punct::Shl => (B(BinaryOp::Shl), 8), + Punct::Shr => (B(BinaryOp::Shr), 8), + Punct::UShr => (B(BinaryOp::UShr), 8), + Punct::Plus => (B(BinaryOp::Add), 9), + Punct::Minus => (B(BinaryOp::Sub), 9), + Punct::Star => (B(BinaryOp::Mul), 10), + Punct::Slash => (B(BinaryOp::Div), 10), + Punct::Percent => (B(BinaryOp::Mod), 10), + _ => return None, + }), + _ => None, + } +} + +fn prefix_op(token: &Token) -> Option { + Some(match &token.kind { + TokenKind::Keyword(Keyword::Delete) => Prefix::Unary(UnaryOp::Delete), + TokenKind::Keyword(Keyword::Void) => Prefix::Unary(UnaryOp::Void), + TokenKind::Keyword(Keyword::Typeof) => Prefix::Unary(UnaryOp::Typeof), + TokenKind::Punct(Punct::Plus) => Prefix::Unary(UnaryOp::Plus), + TokenKind::Punct(Punct::Minus) => Prefix::Unary(UnaryOp::Minus), + TokenKind::Punct(Punct::Tilde) => Prefix::Unary(UnaryOp::BitNot), + TokenKind::Punct(Punct::Bang) => Prefix::Unary(UnaryOp::Not), + TokenKind::Punct(Punct::PlusPlus) => Prefix::Update(UpdateOp::Increment), + TokenKind::Punct(Punct::MinusMinus) => Prefix::Update(UpdateOp::Decrement), + _ => return None, + }) +} + +/// A token as an error message names it. +fn describe(token: &Token) -> String { + match &token.kind { + TokenKind::Eof => "end of input".into(), + TokenKind::Ident(name) => format!("identifier '{name}'"), + TokenKind::Keyword(k) => format!("token '{}'", k.as_str()), + TokenKind::Punct(p) => format!("token '{}'", p.as_str()), + TokenKind::Number(_) => "number".into(), + TokenKind::String(_) => "string".into(), + TokenKind::RegExp { .. } => "regular expression".into(), + TokenKind::Directive(_) => "preprocessor line".into(), + } +} diff --git a/crates/js/src/regexp.rs b/crates/js/src/regexp.rs new file mode 100644 index 000000000..cda4f5307 --- /dev/null +++ b/crates/js/src/regexp.rs @@ -0,0 +1,191 @@ +//! The early check of a regular expression literal's pattern (ECMA-262 5.1 section 15.10.1, with +//! the lenient forms of Annex B that scripts rely on: a `]`, `{` or `}` that starts no class or +//! quantifier is a literal character, a lookahead may take a quantifier, and a class range with a +//! class escape at one end is a set of literal characters). +//! +//! The check reads the pattern once, front to back, counting open groups instead of recursing. + +/// Checks `pattern`, the text between a literal's slashes. Groups may nest `max_depth` deep. +pub fn check(pattern: &[u16], max_depth: usize) -> Result<(), String> { + let mut i = 0; + let mut depth = 0usize; + // The term before the current position may take a quantifier. + let mut quantifiable = false; + while let Some(&unit) = pattern.get(i) { + let c = char::from_u32(u32::from(unit)).unwrap_or('\u{fffd}'); + match c { + '\\' => { + let next = pattern.get(i + 1).ok_or("\\ at end of pattern")?; + quantifiable = !matches!(char::from_u32(u32::from(*next)), Some('b' | 'B')); + i += 2; + } + '(' => { + if pattern.get(i + 1) == Some(&u16::from(b'?')) { + match pattern.get(i + 2).and_then(|&u| char::from_u32(u32::from(u))) { + Some(':' | '=' | '!') => i += 3, + _ => return Err("invalid group".into()), + } + } else { + i += 1; + } + depth += 1; + if depth > max_depth { + return Err(crate::error::TOO_DEEP.into()); + } + quantifiable = false; + } + ')' => { + depth = depth.checked_sub(1).ok_or("unmatched ')'")?; + quantifiable = true; + i += 1; + } + '[' => { + i = class(pattern, i + 1)?; + quantifiable = true; + } + '*' | '+' | '?' => { + if !quantifiable { + return Err("nothing to repeat".into()); + } + i = lazy(pattern, i + 1); + quantifiable = false; + } + '{' => match braces(pattern, i + 1) { + Some((min, max, end)) => { + if !quantifiable { + return Err("nothing to repeat".into()); + } + if max.is_some_and(|max| max < min) { + return Err("numbers out of order in {} quantifier".into()); + } + i = lazy(pattern, end); + quantifiable = false; + } + None => { + quantifiable = true; + i += 1; + } + }, + '^' | '$' | '|' => { + quantifiable = false; + i += 1; + } + _ => { + quantifiable = true; + i += 1; + } + } + } + if depth > 0 { + return Err("unterminated group".into()); + } + Ok(()) +} + +/// After a quantifier ending before `i`: past the `?` that makes it lazy, if there is one. +fn lazy(pattern: &[u16], i: usize) -> usize { + if pattern.get(i) == Some(&u16::from(b'?')) { i + 1 } else { i } +} + +/// A `{n}`, `{n,}` or `{n,m}` quantifier whose `{` is before `i`: its bounds and the position +/// after its `}`. `None` when the text there is not one. +fn braces(pattern: &[u16], i: usize) -> Option<(u64, Option, usize)> { + let (min, mut i) = digits(pattern, i)?; + let max = if pattern.get(i) == Some(&u16::from(b',')) { + i += 1; + match digits(pattern, i) { + Some((max, end)) => { + i = end; + Some(max) + } + None => None, + } + } else { + Some(min) + }; + (pattern.get(i) == Some(&u16::from(b'}'))).then_some((min, max, i + 1)) +} + +/// Decimal digits at `i`: their value (saturated) and the position after them. +fn digits(pattern: &[u16], mut i: usize) -> Option<(u64, usize)> { + let start = i; + let mut value: u64 = 0; + while let Some(d) = pattern.get(i).and_then(|&u| char::from_u32(u32::from(u))).and_then(|c| c.to_digit(10)) { + value = value.saturating_mul(10).saturating_add(u64::from(d)); + i += 1; + } + (i > start).then_some((value, i)) +} + +/// Reads a character class whose `[` is before `i` and returns the position after its `]`. +fn class(pattern: &[u16], mut i: usize) -> Result { + if pattern.get(i) == Some(&u16::from(b'^')) { + i += 1; + } + // The last class atom's character (`None` for a class escape such as `\d`), when a `-` after it + // may make a range. + let mut prev: Option> = None; + loop { + let unit = *pattern.get(i).ok_or("unterminated character class")?; + if unit == u16::from(b']') { + return Ok(i + 1); + } + let (atom, next) = class_atom(pattern, i)?; + i = next; + let dash = pattern.get(i) == Some(&u16::from(b'-')) && pattern.get(i + 1).is_some_and(|&u| u != u16::from(b']')); + match prev.take() { + Some(low) => { + if let (Some(low), Some(high)) = (low, atom) + && low > high + { + return Err("range out of order in character class".into()); + } + } + None if dash => { + prev = Some(atom); + i += 1; + } + None => {} + } + } +} + +/// A class atom at `i`: its character (`None` for a class escape) and the position after it. +fn class_atom(pattern: &[u16], i: usize) -> Result<(Option, usize), String> { + let unit = u32::from(*pattern.get(i).ok_or("unterminated character class")?); + if unit != u32::from(b'\\') { + return Ok((Some(unit), i + 1)); + } + let next = u32::from(*pattern.get(i + 1).ok_or("\\ at end of pattern")?); + let c = char::from_u32(next).unwrap_or('\u{fffd}'); + let hex = |n: usize| -> Option { + (0..n).try_fold(0, |v, k| pattern.get(i + 2 + k).and_then(|&u| crate::chars::hex_value(u32::from(u))).map(|d| v * 16 + d)) + }; + Ok(match c { + 'd' | 'D' | 's' | 'S' | 'w' | 'W' => (None, i + 2), + 'b' => (Some(0x08), i + 2), + 'f' => (Some(0x0C), i + 2), + 'n' => (Some(0x0A), i + 2), + 'r' => (Some(0x0D), i + 2), + 't' => (Some(0x09), i + 2), + 'v' => (Some(0x0B), i + 2), + 'x' => hex(2).map_or((Some(next), i + 2), |v| (Some(v), i + 4)), + 'u' => hex(4).map_or((Some(next), i + 2), |v| (Some(v), i + 6)), + 'c' => match pattern.get(i + 2).map(|&u| u32::from(u)) { + Some(l) if char::from_u32(l).is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_') => (Some(l % 32), i + 3), + _ => (Some(u32::from(b'\\')), i + 1), + }, + '0'..='7' => { + let mut value = 0; + let mut j = i + 1; + while j < i + 4 + && let Some(d) = pattern.get(j).and_then(|&u| char::from_u32(u32::from(u))).and_then(|ch| ch.to_digit(8)) + { + value = value * 8 + d; + j += 1; + } + (Some(value), j) + } + _ => (Some(next), i + 2), + }) +} diff --git a/crates/js/src/tests.rs b/crates/js/src/tests.rs new file mode 100644 index 000000000..fb357e1bf --- /dev/null +++ b/crates/js/src/tests.rs @@ -0,0 +1,804 @@ +use proptest::prelude::*; + +use crate::{ + BinaryOp, Expr, ExprId, ForInTarget, FunctionKind, Options, Program, PropertyValue, Stmt, StmtId, SyntaxError, TOO_DEEP, TokenKind, parse, + tokenize, +}; + +fn ok(src: &str) -> Program { + match parse(src, &Options::default()) { + Ok(p) => p, + Err(e) => panic!("{src:?} should parse: {e}"), + } +} + +fn err(src: &str) -> SyntaxError { + match parse(src, &Options::default()) { + Ok(_) => panic!("{src:?} should not parse"), + Err(e) => e, + } +} + +fn strict_err(src: &str) -> SyntaxError { + let full = format!("'use strict';\n{src}"); + match parse(&full, &Options::default()) { + Ok(_) => panic!("{src:?} should not parse in strict mode code"), + Err(e) => e, + } +} + +fn with_preprocessor() -> Options { + Options { preprocessor_lines: true, ..Options::default() } +} + +/// The expression of the program's only (or first) statement. +fn first_expr(p: &Program) -> ExprId { + match p.ast.stmt(p.body[0]) { + Some(Stmt::Expr(e)) => *e, + other => panic!("not an expression statement: {other:?}"), + } +} + +fn expr(p: &Program, id: ExprId) -> &Expr { + p.ast.expr(id).unwrap() +} + +fn stmt(p: &Program, id: StmtId) -> &Stmt { + p.ast.stmt(id).unwrap() +} + +fn utf16(s: &str) -> Vec { + s.encode_utf16().collect() +} + +// ---- expressions ---- + +#[test] +fn binary_operators_follow_precedence_and_associate_left() { + let p = ok("a + b * c - d;"); + let Expr::Binary { op: BinaryOp::Sub, left, right } = expr(&p, first_expr(&p)) else { panic!() }; + assert!(matches!(expr(&p, *right), Expr::Ident(n) if n == "d")); + let Expr::Binary { op: BinaryOp::Add, right: mul, .. } = expr(&p, *left) else { panic!() }; + assert!(matches!(expr(&p, *mul), Expr::Binary { op: BinaryOp::Mul, .. })); +} + +#[test] +fn logical_operators_bind_looser_than_comparisons() { + let p = ok("a || b && c == d;"); + let Expr::Logical { right, .. } = expr(&p, first_expr(&p)) else { panic!() }; + let Expr::Logical { right: eq, .. } = expr(&p, *right) else { panic!() }; + assert!(matches!(expr(&p, *eq), Expr::Binary { op: BinaryOp::Eq, .. })); +} + +#[test] +fn assignment_associates_right() { + let p = ok("a = b += c;"); + let Expr::Assign { op: None, value, .. } = expr(&p, first_expr(&p)) else { panic!() }; + assert!(matches!(expr(&p, *value), Expr::Assign { op: Some(BinaryOp::Add), .. })); +} + +#[test] +fn new_takes_the_first_argument_list_after_its_callee() { + let p = ok("new a.b(1).c(2);"); + let Expr::Call { callee, args } = expr(&p, first_expr(&p)) else { panic!() }; + assert_eq!(args.len(), 1); + let Expr::Member { object, property } = expr(&p, *callee) else { panic!() }; + assert_eq!(property.name, "c"); + let Expr::New { callee, args } = expr(&p, *object) else { panic!() }; + assert_eq!(args.len(), 1); + assert!(matches!(expr(&p, *callee), Expr::Member { .. })); + + let p = ok("new new X()();"); + let Expr::New { callee, .. } = expr(&p, first_expr(&p)) else { panic!() }; + assert!(matches!(expr(&p, *callee), Expr::New { .. })); + + let p = ok("new X;"); + assert!(matches!(expr(&p, first_expr(&p)), Expr::New { args, .. } if args.is_empty())); +} + +#[test] +fn conditional_and_sequence_expressions() { + let p = ok("a ? b : c ? d : e, f;"); + let Expr::Sequence(list) = expr(&p, first_expr(&p)) else { panic!() }; + assert_eq!(list.len(), 2); + let Expr::Conditional { alternate, .. } = expr(&p, list[0]) else { panic!() }; + assert!(matches!(expr(&p, *alternate), Expr::Conditional { .. })); +} + +#[test] +fn array_literals_keep_holes() { + let p = ok("[1, , 3, ];"); + let Expr::Array(elements) = expr(&p, first_expr(&p)) else { panic!() }; + assert_eq!(elements.len(), 3); + assert!(elements[1].is_none()); + let p = ok("[,];"); + assert!(matches!(expr(&p, first_expr(&p)), Expr::Array(e) if e.len() == 1)); +} + +#[test] +fn object_literals_with_accessors_and_keyword_names() { + let p = ok("({a: 1, 'b': 2, 3: 3, if: 4, get: 5, set: 6, get x() { return 1; }, set x(v) {}, });"); + let Expr::Paren(inner) = expr(&p, first_expr(&p)) else { panic!() }; + let Expr::Object(props) = expr(&p, *inner) else { panic!() }; + assert_eq!(props.len(), 8); + assert!(matches!(props[6].value, PropertyValue::Get(_))); + assert!(matches!(props[7].value, PropertyValue::Set(_))); + let PropertyValue::Get(getter) = props[6].value else { panic!() }; + assert_eq!(p.ast.function(getter).unwrap().kind, FunctionKind::Getter); +} + +#[test] +fn accessors_take_the_right_number_of_parameters() { + assert!(err("({get a(x) {}});").message.contains("getter")); + assert!(err("({set a() {}});").message.contains("setter")); + assert!(err("({set a(x, y) {}});").message.contains("setter")); +} + +#[test] +fn property_names_may_repeat() { + for src in ["({a: 1, a: 2});", "({a: 1, 'a': 2});", "({get a() {}, set a(v) {}});", "({a: 1, get a() {}});", "({get a() {}, get a() {}});"] { + ok(src); + ok(&format!("'use strict'; {src}")); + } +} + +#[test] +fn escaped_get_and_set_do_not_start_accessors() { + assert!(err("({g\\u0065t a() {}});").message.contains("expected ':'")); + assert!(err("({\\u0073et a(v) {}});").message.contains("expected ':'")); + ok("({g\\u0065t: 1});"); +} + +#[test] +fn member_names_may_be_reserved_words() { + ok("a.if.var.function = b.true;"); + ok("a.\\u0076ar = 1;"); +} + +// ---- statements ---- + +#[test] +fn else_if_chains_nest_as_alternates() { + let p = ok("if (a) x; else if (b) y; else z;"); + let Stmt::If { alternate: Some(alt), .. } = stmt(&p, p.body[0]) else { panic!() }; + let Stmt::If { alternate: Some(last), .. } = stmt(&p, *alt) else { panic!() }; + assert!(matches!(stmt(&p, *last), Stmt::Expr(_))); +} + +#[test] +fn for_statements_of_every_form() { + ok("for (;;) break;"); + ok("for (var i = 0, n = a.length; i < n; i++) {}"); + ok("for (i = 0; i < 10; i++) continue;"); + ok("for (var k in o) {}"); + ok("for (k in o) {}"); + ok("for (a.b in o) {}"); + ok("for (var x = 1 in o) {}"); + ok("for (var i = (a in b); i; ) break;"); + let p = ok("for (var k in o) {}"); + assert!(matches!(stmt(&p, p.body[0]), Stmt::ForIn { target: ForInTarget::Var(_), .. })); + assert!(err("for (a + b in o) {}").message.contains("for-in")); + assert!(err("for (var a, b in o) {}").message.contains("expected ';'")); +} + +#[test] +fn switch_try_with_labels_and_debugger() { + ok("switch (x) { case 1: a(); break; default: b(); case 2: }"); + ok("try { a(); } catch (e) { b(); } finally { c(); }"); + ok("try {} finally {}"); + ok("with (o) { a = 1; }"); + ok("outer: for (;;) { inner: for (;;) { continue outer; } }"); + ok("a: b: while (1) { continue a; }"); + ok("a: { break a; }"); + ok("debugger;"); + ok("do x(); while (a) y();"); + assert!(err("switch (x) { default: default: }").message.contains("default")); + assert!(err("try {}").message.contains("catch or finally")); +} + +#[test] +fn break_and_continue_need_a_target() { + assert!(err("break;").message.contains("break outside")); + assert!(err("continue;").message.contains("continue outside")); + assert!(err("switch (x) { case 1: continue; }").message.contains("continue outside")); + assert!(err("for (;;) { break nowhere; }").message.contains("undefined label")); + assert!(err("a: { continue a; }").message.contains("does not name a loop")); + assert!(err("a: while (1) { function f() { break a; } }").message.contains("undefined label")); + assert!(err("while (1) { function f() { break; } }").message.contains("break outside")); + assert!(err("a: a: ;").message.contains("duplicate label")); + assert!(err("a: { a: ; }").message.contains("duplicate label")); + ok("a: ; a: ;"); +} + +#[test] +fn return_only_inside_functions() { + assert!(err("return 1;").message.contains("return outside")); + ok("function f() { return; }"); + ok("(function () { return 1 })();"); +} + +#[test] +fn function_declarations_in_blocks_and_branches() { + ok("{ function f() {} }"); + ok("if (a) function f() {} else function g() {}"); + ok("switch (a) { case 1: function f() {} }"); + ok("l: function f() {}"); + ok("{ l: m: function f() {} }"); + ok("'use strict'; { function f() {} }"); + assert!(strict_err("if (a) function f() {}").message.contains("functions can only be declared")); + assert!(strict_err("l: function f() {}").message.contains("functions can only be declared")); + for body in ["while (a) function f() {}", "for (;;) function f() {}", "do function f() {} while (a)", "with (a) function f() {}"] { + assert!(err(body).message.contains("cannot stand here"), "{body}"); + } + assert!(err("if (a) l: function f() {}").message.contains("cannot stand here")); + assert!(err("while (a) l: function f() {}").message.contains("cannot stand here")); +} + +// ---- automatic semicolon insertion ---- + +#[test] +fn semicolons_are_inserted_at_line_breaks_braces_and_the_end() { + ok("a = 1\nb = 2"); + ok("function f() { return 1 }"); + ok("a = 1"); + let e = err("a = 1 b = 2"); + assert_eq!((e.line, e.column), (1, 7)); +} + +#[test] +fn restricted_productions_end_at_a_line_break() { + let p = ok("function f() { return\n1 }"); + let Stmt::FunctionDecl(f) = stmt(&p, p.body[0]) else { panic!() }; + let body = &p.ast.function(*f).unwrap().body; + assert_eq!(body.len(), 2); + assert!(matches!(stmt(&p, body[0]), Stmt::Return(None))); + + let p = ok("a\n++b"); + assert_eq!(p.body.len(), 2); + assert!(matches!(expr(&p, first_expr(&p)), Expr::Ident(_))); + + ok("for (;;) { break\nfoo }"); + assert!(err("throw\nnew Error()").message.contains("line break after throw")); +} + +#[test] +fn no_semicolon_is_inserted_inside_a_for_header() { + assert!(err("for (a\nb\n) {}").message.contains("expected ';'")); +} + +// ---- the division and regular expression goals ---- + +#[test] +fn slash_divides_after_a_value_and_starts_a_regexp_elsewhere() { + let p = ok("a / b / c;"); + assert!(matches!(expr(&p, first_expr(&p)), Expr::Binary { op: BinaryOp::Div, .. })); + let p = ok("x = /a\\/b[/]c/gi;"); + let Expr::Assign { value, .. } = expr(&p, first_expr(&p)) else { panic!() }; + let Expr::RegExp { pattern, flags } = expr(&p, *value) else { panic!() }; + assert_eq!(*pattern, utf16("a\\/b[/]c")); + assert_eq!(flags, "gi"); + ok("if (x) /re/.test(y);"); + ok("{} /re/.test(y);"); + ok("x = y /= 2;"); + ok("x = /=a/;"); + ok("return_ = typeof /x/;"); + ok("a(/x/, /y/);"); + // After a line break a `/` still divides when it can: there is no semicolon to insert. + let p = ok("a = b\n/c/g"); + assert_eq!(p.body.len(), 1); +} + +#[test] +fn tokenize_reads_each_slash_as_the_grammar_decides() { + let tokens = tokenize("a = b / c; d = /e/g.exec(f);", &Options::default()).unwrap(); + let kinds: Vec<&TokenKind> = tokens.iter().map(|t| &t.kind).collect(); + assert!(kinds.iter().any(|k| matches!(k, TokenKind::RegExp { flags, .. } if flags == "g"))); + assert_eq!(kinds.iter().filter(|k| matches!(k, TokenKind::RegExp { .. })).count(), 1); + assert!(matches!(kinds.last(), Some(TokenKind::Eof))); +} + +// ---- literals ---- + +fn number(src: &str) -> f64 { + let p = ok(src); + let Expr::Number(n) = expr(&p, first_expr(&p)) else { panic!("{src}") }; + *n +} + +fn string(src: &str) -> Vec { + let p = ok(src); + let Expr::String(s) = expr(&p, first_expr(&p)) else { panic!("{src}") }; + s.clone() +} + +#[test] +fn numeric_literals() { + assert_eq!(number("0;"), 0.0); + assert_eq!(number("12.5e-1;"), 1.25); + assert_eq!(number(".5;"), 0.5); + assert_eq!(number("5.;"), 5.0); + assert_eq!(number("1E3;"), 1000.0); + assert_eq!(number("0x1F;"), 31.0); + assert_eq!(number("0XfF;"), 255.0); + assert_eq!(number("0x20000000000001;"), 9007199254740992.0); + assert_eq!(number("017;"), 15.0); + assert_eq!(number("08;"), 8.0); + assert_eq!(number("09.5;"), 9.5); + assert_eq!(number("1e400;"), f64::INFINITY); + assert!(err("3in []").message.contains("immediately after numeric literal")); + assert!(err("0x;").message.contains("hexadecimal")); + assert!(err("1.e").message.contains("immediately after")); + assert!(strict_err("017;").message.contains("octal literals")); + assert!(strict_err("08;").message.contains("octal literals")); + assert!(strict_err("({017: 1});").message.contains("octal")); +} + +#[test] +fn string_literals_and_escapes() { + assert_eq!(string(r#""a\tb\n\\\"\'";"#), utf16("a\tb\n\\\"'")); + assert_eq!(string(r#"'\x41é€';"#), utf16("A\u{e9}\u{20ac}")); + assert_eq!(string("'a\\\nb';"), utf16("ab")); + assert_eq!(string("'a\\\r\nb';"), utf16("ab")); + assert_eq!(string(r"'\0';"), vec![0]); + assert_eq!(string(r"'\101\7\08';"), vec![0x41, 7, 0, u16::from(b'8')]); + assert_eq!(string(r"'\q\8';"), utf16("q8")); + assert_eq!(string("'\u{2028}';"), vec![0x2028]); + assert_eq!(string("'\u{1F600}';"), utf16("\u{1F600}")); + assert!(err("'abc").message.contains("unterminated string")); + assert!(err("'a\nb'").message.contains("unterminated string")); + assert!(err(r"'\x4';").message.contains("hexadecimal escape")); + assert!(err(r"'\u12';").message.contains("Unicode escape")); + assert_eq!(string(r"'\u{41}\u{1F600}\u{0000E9}';"), utf16("A\u{1F600}\u{e9}")); + for bad in [r"'\u{}';", r"'\u{110000}';", r"'\u{41';", r"'\u{4g}';"] { + assert!(err(bad).message.contains("Unicode escape"), "{bad}"); + } + assert!(strict_err(r"'\101';").message.contains("octal escape")); + assert!(strict_err(r"'\8';").message.contains("octal escape")); + ok(r"'use strict'; '\0';"); +} + +#[test] +fn identifiers_with_unicode_and_escapes() { + let p = ok("var caf\\u00e9 = 1, \u{3b1}\u{3b2} = 2, $_x1 = 3, a\u{300} = 4;"); + let Stmt::Var(decls) = stmt(&p, p.body[0]) else { panic!() }; + let names: Vec<&str> = decls.iter().map(|d| d.name.name.as_str()).collect(); + assert_eq!(names, ["caf\u{e9}", "\u{3b1}\u{3b2}", "$_x1", "a\u{300}"]); + let p = ok("\\u{61}b;"); + assert!(matches!(expr(&p, first_expr(&p)), Expr::Ident(n) if n == "ab")); + assert!(err("var \\u0076ar = 1;").message.contains("escaped characters")); + assert!(err("var a\\u0020b;").message.contains("identifier escape")); + assert!(err("var \\x41;").message.contains("escape")); + assert!(err("var a = 1 @ 2;").message.contains("unexpected character '@'")); + assert!(err("var \u{2e2f};").message.contains("unexpected character")); + assert!(err("var a\u{2e2f};").message.contains("unexpected character")); +} + +#[test] +fn identifier_characters_follow_their_unicode_categories() { + // Letters (Lu, Ll, Lt, Lm, Lo, Nl) start an identifier; marks (Mn, Mc), decimal digits (Nd) + // and connector punctuation (Pc) may follow. + for good in [ + "var \u{995}\u{9cd}\u{9b7} = 1;", + "var \u{b95}\u{bcd}\u{b95} = 1;", + "var a\u{9be} = 1;", + "var x\u{966} = 1;", + "var a\u{203f}b = 1;", + "var \u{2b0} = 1;", + "var \u{2160} = 1;", + "var \u{1c5} = 1;", + "var \u{5d0}\u{5b7} = 1;", + ] { + ok(good); + } + // Other numbers (No) are not identifier characters, and a mark cannot start an identifier. + for bad in ["var a\u{b2} = 1;", "var x\u{bd} = 1;", "var \u{9be}a = 1;", "var \u{300}a = 1;", "var \u{966} = 1;"] { + assert!(err(bad).message.contains("unexpected character"), "{bad:?}"); + } + assert!(parse("var \\u09BEa = 1;", &Options::default()).is_err()); + assert!(parse("var a\\u00B2 = 1;", &Options::default()).is_err()); +} + +#[test] +fn comments_and_line_terminators() { + let p = ok("a /* one\ntwo */ b"); + assert_eq!(p.body.len(), 2, "a comment with a line break separates statements"); + assert!(err("a /* one two */ b").message.contains("unexpected identifier 'b'")); + assert!(err("/* open").message.contains("unterminated comment")); + ok("// only a comment"); + ok("a\u{2028}b\u{2029}c\r\nd\re"); + ok("\u{feff}var\u{a0}a\u{3000}=\u{b}1\u{c};"); +} + +// ---- regular expression literals ---- + +#[test] +fn regular_expression_flags_and_patterns_are_checked() { + for good in [ + "/a|b*c+?d{2}e{1,}f{1,3}?/", + "/[a-z\\d-]/", + "/[\\d-z]/", + "/]/", + "/{/", + "/a{,}/", + "/x{a}/", + "/(?:a)(?=b)(?!c)/", + "/(?=a)*/", + "/\\1(a)/", + "/[]/", + "/[^]/", + "/\\cJ\\c/", + "/[\\b-\\n]/", + "/a/gim", + ] { + ok(&format!("x = {good};")); + } + for (bad, message) in [ + ("/(/", "unterminated group"), + ("/a)/", "unmatched"), + ("/?/", "nothing to repeat"), + ("/a**/", "nothing to repeat"), + ("/^*/", "nothing to repeat"), + ("/{1}/", "nothing to repeat"), + ("/a{2,1}/", "out of order"), + ("/[z-a]/", "out of order"), + ("/(?x)/", "invalid group"), + ("/[a/", "unterminated regular expression"), + ("/a\\/", "unterminated regular expression"), + ("/a\n/", "unterminated regular expression"), + ("/a/gg", "flags"), + ("/a/x", "flags"), + ("/a/\\u0067", "flags"), + ] { + let e = err(&format!("x = {bad};")); + assert!(e.message.contains(message), "{bad}: {}", e.message); + assert_eq!((e.line, e.column), (1, 5), "{bad}"); + } +} + +// ---- strict mode ---- + +#[test] +fn use_strict_makes_the_rest_strict_mode_code() { + assert!(ok("'use strict'; a;").strict); + assert!(ok("\"use strict\"\na;").strict); + assert!(!ok("'use\\x20strict'; a;").strict, "a directive with an escape is not \"use strict\""); + assert!(!ok("('use strict'); a;").strict); + assert!(!ok("a; 'use strict';").strict); + assert!(ok("'other'; 'use strict'; a;").strict); + assert!(!ok("'use strict' + 1;").strict); + let p = ok("function f() { 'use strict'; } function g() {}"); + let funcs = &p.ast.functions; + assert!(funcs[0].strict && !funcs[1].strict && !p.strict); + assert!(parse("with (a) {}", &Options { strict: true, ..Options::default() }).is_err()); +} + +#[test] +fn strict_mode_early_errors() { + assert!(strict_err("with (a) {}").message.contains("with statements")); + assert!(strict_err("delete a;").message.contains("delete")); + assert!(strict_err("delete ((a));").message.contains("delete")); + ok("'use strict'; delete a.b; delete a[0];"); + for target in ["eval = 1;", "arguments++;", "--eval;", "arguments += 1;", "(eval) = 1;", "for (eval in o) {}"] { + assert!(strict_err(target).message.contains("cannot assign"), "{target}"); + } + for binding in ["var eval;", "function arguments() {}", "function f(eval) {}", "try {} catch (arguments) {}", "(function eval() {});"] { + assert!(strict_err(binding).message.contains("cannot be declared"), "{binding}"); + } + for word in ["implements", "interface", "let", "package", "private", "protected", "public", "static", "yield"] { + ok(&format!("var {word} = 1;")); + assert!(strict_err(&format!("var {word} = 1;")).message.contains("reserved word"), "{word}"); + assert!(strict_err(&format!("{word};")).message.contains("reserved word"), "{word}"); + } + assert!(strict_err("function f(a, a) {}").message.contains("duplicate parameter")); + ok("function f(a, a) {}"); + let e = strict_err("for (var x = 1 in o) {}"); + assert!(e.message.contains("initializer"), "{}", e.message); + assert_eq!((e.line, e.column), (2, 16)); + ok("'use strict'; for (var x in o) {}"); +} + +#[test] +fn a_function_made_strict_by_its_own_prologue_checks_its_name_and_parameters() { + let e = err("function f(a, a) { 'use strict'; }"); + assert!(e.message.contains("duplicate parameter")); + assert_eq!((e.line, e.column), (1, 15)); + assert!(err("function eval() { 'use strict'; }").message.contains("cannot be declared")); + assert!(err("function f(arguments) { 'use strict'; }").message.contains("cannot be declared")); + assert!(err("function static() { 'use strict'; }").message.contains("reserved word")); + assert!(err("function f() { '\\01'; 'use strict'; }").message.contains("octal escape")); + assert!(err("'\\01'; 'use strict';").message.contains("octal escape")); + assert!(err("function f() { 'use strict'; 010 }").message.contains("octal literals")); + assert!(err("function f() { 'use strict'; var public; }").message.contains("reserved word")); +} + +// ---- assignment targets ---- + +#[test] +fn invalid_assignment_targets() { + for bad in ["1 = 2;", "a + b = c;", "(a, b) = 1;", "a = 1 = 2;", "++1;", "1++;", "--a.b++;", "this = 1;", "'a' += 1;", "(a = 1) = 2;"] { + let e = err(bad); + assert!(e.message.contains("invalid"), "{bad}: {}", e.message); + } + ok("(a) = 1; a.b = 1; a[0] += 2; a.b++; --a[1]; (a.b) = c;"); + // Assigning to a call's result fails at run time outside strict mode code, and is a + // SyntaxError in it. + ok("f() = 1; f()++; for (f() in o) {}"); + for call in ["f() = 1;", "f() += 1;", "++f();", "f()--;", "(f()) = 1;", "for (f() in o) {}"] { + assert!(strict_err(call).message.contains("invalid") || strict_err(call).message.contains("for-in"), "{call}"); + } +} + +// ---- preprocessor lines ---- + +#[test] +fn preprocessor_lines_are_read_as_directives() { + let src = "#target vectorcraft\n #include \"lib/a.jsxinc\"\n//@includepath \"~/x;~/y\"\n#strict on\n#unknown x\nvar a = 1;\n#targetengine main"; + let p = parse(src, &with_preprocessor()).unwrap(); + let names: Vec<(&str, u32, &str)> = p.directives.iter().map(|d| (d.name.as_str(), d.line, d.value.as_str())).collect(); + assert_eq!( + names, + [ + ("target", 1, "vectorcraft"), + ("include", 2, "\"lib/a.jsxinc\""), + ("includepath", 3, "\"~/x;~/y\""), + ("strict", 4, "on"), + ("unknown", 5, "x"), + ("targetengine", 7, "main"), + ] + ); + assert_eq!(p.directives[2].form, "//@"); + assert!(p.directives[4].known.is_none()); + assert_eq!(p.body.len(), 1); + assert!(parse(src, &Options::default()).is_err(), "without preprocessor lines a # is an error"); +} + +#[test] +fn a_hash_away_from_the_start_of_a_line_is_an_error() { + let e = parse("var a = 1; #include x", &with_preprocessor()).unwrap_err(); + assert!(e.message.contains("unexpected character '#'")); + assert_eq!((e.line, e.column), (1, 12)); + assert!(parse("/* c */ #include x", &with_preprocessor()).is_err()); + // A comment naming no directive stays a comment. + assert!(parse("//@other\n", &with_preprocessor()).unwrap().directives.is_empty()); +} + +#[test] +fn a_directive_between_tokens_keeps_its_place_in_the_token_list() { + let tokens = tokenize("a\n#include x\nb", &with_preprocessor()).unwrap(); + let kinds: Vec = tokens + .iter() + .map(|t| match &t.kind { + TokenKind::Ident(n) => n.clone(), + TokenKind::Directive(d) => format!("#{}", d.name), + TokenKind::Eof => "eof".into(), + other => format!("{other:?}"), + }) + .collect(); + assert_eq!(kinds, ["a", "#include", "b", "eof"]); + assert!(tokens[2].newline_before); +} + +// ---- positions ---- + +#[test] +fn errors_carry_line_and_column() { + let e = err("var a = 1;\nvar b = ];"); + assert_eq!((e.line, e.column, e.offset), (2, 9, 19)); + assert_eq!(e.to_string(), "SyntaxError: unexpected token ']' (line 2, column 9)"); + let e = err("a\r\nb\r\n 'x"); + assert_eq!((e.line, e.column), (3, 3)); + let e = err("a\u{2028} )"); + assert_eq!((e.line, e.column), (2, 3)); + let e = err("'\u{1F600}' )"); + assert_eq!((e.line, e.column), (1, 6), "columns count UTF-16 code units"); + let e = err("x = 'a\\\nb' )"); + assert_eq!((e.line, e.column), (2, 4), "a line continuation in a string counts its line"); +} + +#[test] +fn line_count_and_spans() { + let p = ok("a;\r\nb;\nc;"); + assert_eq!(p.line_count(), 3); + assert_eq!(ok("a;\n").line_count(), 1); + assert_eq!(ok("").line_count(), 0); + let span = p.ast.stmt_span(p.body[2]); + assert_eq!(p.line_column(span.start), (3, 1)); + assert_eq!((span.start, span.end), (7, 9)); +} + +// ---- nesting and limits ---- + +#[test] +fn deep_nesting_gives_a_syntax_error_not_a_stack_overflow() { + const N: usize = 100_000; + let cases = [ + format!("{}1{}", "(".repeat(N), ")".repeat(N)), + format!("{}1{}", "[".repeat(N), "]".repeat(N)), + format!("x = {}{}", "{a:".repeat(N), "}".repeat(N)), + format!("{}{}", "{".repeat(N), "}".repeat(N)), + format!("{}{}", "function f(){".repeat(N), "}".repeat(N)), + format!("{}1", "a ? b : ".repeat(N)), + format!("{}x{}", "f(".repeat(N), ")".repeat(N)), + format!("{}x", "if (a) ".repeat(N)), + ]; + for src in &cases { + let e = err(src); + assert_eq!(e.message, TOO_DEEP, "{}", &src[..20]); + assert_eq!(e.line, 1); + assert!(e.column > 1); + } + let e = err(&format!("x = /{}a{}/;", "(".repeat(N), ")".repeat(N))); + assert!(e.message.contains(TOO_DEEP)); +} + +#[test] +fn nesting_up_to_the_limit_parses() { + let depth = 50; + ok(&format!("{}1{};", "(".repeat(depth), ")".repeat(depth))); + ok(&format!("{}{}", "{".repeat(depth), "}".repeat(depth))); + ok(&format!("x = {}{};", "[".repeat(depth), "]".repeat(depth))); +} + +#[test] +fn long_flat_runs_do_not_nest() { + const N: usize = 100_000; + ok(&format!("x = 1{};", " + 1".repeat(N))); + ok(&format!("x = {}y;", "!".repeat(N))); + ok(&format!("{}x = 1;", "a = ".repeat(N))); + ok(&format!("if (a) b;{} else c;", " else if (a) b;".repeat(N))); + ok(&format!("{}x;", (0..N).map(|i| format!("l{i}: ")).collect::())); + ok(&format!("x{};", ".y".repeat(N))); + ok(&format!("x{};", "()".repeat(N))); + ok(&"a;\n".repeat(N)); +} + +#[test] +fn long_label_chains_and_parameter_lists_are_checked_in_linear_time() { + // Each check looks a name up in a hash set or map. Comparing every name with every earlier + // one would take about 2 * 10^10 steps at this size. + const N: usize = 200_000; + let labels: String = (0..N).map(|i| format!("l{i}: ")).collect(); + let body: String = (0..N).map(|i| format!("m{i}: break l{i}; ")).collect(); + ok(&format!("{labels}for (;;) {{ {body}continue l0; }}")); + assert!(err(&format!("{labels}l0: x;")).message.contains("duplicate label")); + assert!(err(&format!("{labels}for (;;) {{ m: break l{N}; }}")).message.contains("undefined label")); + let params = (0..N).map(|i| format!("p{i}")).collect::>().join(", "); + ok(&format!("function f({params}) {{ 'use strict'; }}")); + assert!(err(&format!("function f({params}, p0) {{ 'use strict'; }}")).message.contains("duplicate parameter")); +} + +#[test] +fn nesting_to_the_default_limit_fits_a_small_stack() { + // The deepest nesting each production allows, parsed on a thread with a 1 MiB stack (the + // stack a wasm build or a secondary thread may have). + let n = crate::DEFAULT_MAX_DEPTH; + let cases = [ + format!("x = {}1{};", "(".repeat(n), ")".repeat(n)), + format!("x = {}1{};", "[".repeat(n), "]".repeat(n)), + format!("x = {}1{};", "{a:".repeat(n), "}".repeat(n)), + format!("{}{}", "function f(){".repeat(n), "}".repeat(n)), + format!("{}x;", "if (a) ".repeat(n)), + format!("x = {}1;", "a ? b : ".repeat(n)), + format!("{}x{};", "f(".repeat(n), ")".repeat(n)), + format!("{}x{};", "a[".repeat(n), "]".repeat(n)), + ]; + let handle = std::thread::Builder::new() + .stack_size(1 << 20) + .spawn(move || cases.iter().map(|src| parse(src, &Options::default()).map(|_| ()).map_err(|e| e.message)).collect::>()) + .unwrap(); + for result in handle.join().unwrap() { + // Each either parses or stops at the limit; neither overflows the stack. + assert!(result.is_ok() || result == Err(TOO_DEEP.to_string()), "{result:?}"); + } +} + +#[test] +fn sources_over_the_length_limit_are_refused() { + let options = Options { max_source_units: 10, ..Options::default() }; + assert_eq!(parse("a = 1;", &options).map(|_| ()), Ok(())); + assert_eq!(parse("a = 1 + 2 + 3;", &options).unwrap_err().message, crate::TOO_LONG); + assert_eq!(parse("'\u{e9}\u{e9}\u{e9}\u{e9}';", &options).map(|_| ()), Ok(()), "the limit counts UTF-16 code units"); +} + +// ---- robustness ---- + +/// Fragments of the language, put together at random so the parser reaches its deeper states. +const FRAGMENTS: &[&str] = &[ + "var ", + "function ", + "f", + "(", + ")", + "{", + "}", + "[", + "]", + ";", + ",", + "=", + "+", + "-", + "/", + "/=", + "*", + "?", + ":", + "!", + "++", + "--", + "if", + "else", + "for", + "in", + "while", + "do", + "return", + "break", + "continue", + "new", + "delete", + "typeof", + "this", + "with", + "switch", + "case", + "default", + "try", + "catch", + "finally", + "throw", + "get", + "set", + "'s'", + "\"\\x4", + "1", + "0x", + "017", + ".5", + "1e", + "/re/g", + "a:", + "\n", + " ", + "'use strict';", + "#include x\n", + "//c\n", + "/*", + "*/", + "\\u0061", + "\u{2028}", + "eval", + "arguments", + ".", + "x", +]; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn arbitrary_text_never_panics(src in ".{0,200}") { + let _ = parse(&src, &Options::default()); + let _ = parse(&src, &Options { strict: true, preprocessor_lines: true, ..Options::default() }); + } + + #[test] + fn arbitrary_fragments_never_panic(parts in proptest::collection::vec(0..FRAGMENTS.len(), 0..80)) { + let src: String = parts.iter().map(|&i| FRAGMENTS[i]).collect(); + for options in [Options::default(), Options { strict: true, preprocessor_lines: true, max_depth: 8, ..Options::default() }] { + match parse(&src, &options) { + Ok(p) => { + for (_, span) in &p.ast.exprs { + prop_assert!(span.start <= span.end && span.end as usize <= src.encode_utf16().count()); + } + } + Err(e) => prop_assert!(e.line >= 1 && e.column >= 1), + } + } + } + + #[test] + fn arbitrary_utf16_never_panics(units in proptest::collection::vec(any::(), 0..120)) { + let _ = crate::parse_utf16(&units, &Options::default()); + } +} diff --git a/crates/js/src/token.rs b/crates/js/src/token.rs new file mode 100644 index 000000000..53bbb35af --- /dev/null +++ b/crates/js/src/token.rs @@ -0,0 +1,391 @@ +//! Tokens: what the lexer gives the parser. + +/// A range of the source in UTF-16 code units, `start` included and `end` not. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Span { + pub start: u32, + pub end: u32, +} + +impl Span { + pub fn new(start: u32, end: u32) -> Self { + Self { start, end } + } + + /// The span from the start of `self` to the end of `other`. + pub fn to(self, other: Span) -> Span { + Span { start: self.start, end: other.end.max(self.start) } + } +} + +/// The reserved words of ECMA-262 5.1 section 7.6.1 that are reserved in all code, with the +/// literals `null`, `true` and `false`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Keyword { + Break, + Case, + Catch, + Class, + Const, + Continue, + Debugger, + Default, + Delete, + Do, + Else, + Enum, + Export, + Extends, + False, + Finally, + For, + Function, + If, + Import, + In, + Instanceof, + New, + Null, + Return, + Super, + Switch, + This, + Throw, + True, + Try, + Typeof, + Var, + Void, + While, + With, +} + +impl Keyword { + /// The keyword spelled `s`, if `s` is one. + pub fn from_word(s: &str) -> Option { + use Keyword as K; + Some(match s { + "break" => K::Break, + "case" => K::Case, + "catch" => K::Catch, + "class" => K::Class, + "const" => K::Const, + "continue" => K::Continue, + "debugger" => K::Debugger, + "default" => K::Default, + "delete" => K::Delete, + "do" => K::Do, + "else" => K::Else, + "enum" => K::Enum, + "export" => K::Export, + "extends" => K::Extends, + "false" => K::False, + "finally" => K::Finally, + "for" => K::For, + "function" => K::Function, + "if" => K::If, + "import" => K::Import, + "in" => K::In, + "instanceof" => K::Instanceof, + "new" => K::New, + "null" => K::Null, + "return" => K::Return, + "super" => K::Super, + "switch" => K::Switch, + "this" => K::This, + "throw" => K::Throw, + "true" => K::True, + "try" => K::Try, + "typeof" => K::Typeof, + "var" => K::Var, + "void" => K::Void, + "while" => K::While, + "with" => K::With, + _ => return None, + }) + } + + pub fn as_str(self) -> &'static str { + use Keyword as K; + match self { + K::Break => "break", + K::Case => "case", + K::Catch => "catch", + K::Class => "class", + K::Const => "const", + K::Continue => "continue", + K::Debugger => "debugger", + K::Default => "default", + K::Delete => "delete", + K::Do => "do", + K::Else => "else", + K::Enum => "enum", + K::Export => "export", + K::Extends => "extends", + K::False => "false", + K::Finally => "finally", + K::For => "for", + K::Function => "function", + K::If => "if", + K::Import => "import", + K::In => "in", + K::Instanceof => "instanceof", + K::New => "new", + K::Null => "null", + K::Return => "return", + K::Super => "super", + K::Switch => "switch", + K::This => "this", + K::Throw => "throw", + K::True => "true", + K::Try => "try", + K::Typeof => "typeof", + K::Var => "var", + K::Void => "void", + K::While => "while", + K::With => "with", + } + } +} + +/// The words reserved only in strict mode code (ECMA-262 5.1 section 7.6.1.2). +pub fn strict_reserved(name: &str) -> bool { + matches!(name, "implements" | "interface" | "let" | "package" | "private" | "protected" | "public" | "static" | "yield") +} + +/// The punctuators of ECMA-262 5.1 sections 7.7 and 7.8.5 (division). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Punct { + LBrace, + RBrace, + LParen, + RParen, + LBracket, + RBracket, + Dot, + Semicolon, + Comma, + Lt, + Gt, + LtEq, + GtEq, + EqEq, + NotEq, + EqEqEq, + NotEqEq, + Plus, + Minus, + Star, + Percent, + PlusPlus, + MinusMinus, + Shl, + Shr, + UShr, + Amp, + Pipe, + Caret, + Bang, + Tilde, + AmpAmp, + PipePipe, + Question, + Colon, + Eq, + PlusEq, + MinusEq, + StarEq, + PercentEq, + ShlEq, + ShrEq, + UShrEq, + AmpEq, + PipeEq, + CaretEq, + Slash, + SlashEq, +} + +impl Punct { + /// Punctuators by spelling, longest first, so the first that matches is the longest. + pub(crate) const ALL: &'static [(&'static str, Punct)] = &[ + (">>>=", Punct::UShrEq), + ("===", Punct::EqEqEq), + ("!==", Punct::NotEqEq), + (">>>", Punct::UShr), + ("<<=", Punct::ShlEq), + (">>=", Punct::ShrEq), + ("<=", Punct::LtEq), + (">=", Punct::GtEq), + ("==", Punct::EqEq), + ("!=", Punct::NotEq), + ("++", Punct::PlusPlus), + ("--", Punct::MinusMinus), + ("<<", Punct::Shl), + (">>", Punct::Shr), + ("&&", Punct::AmpAmp), + ("||", Punct::PipePipe), + ("+=", Punct::PlusEq), + ("-=", Punct::MinusEq), + ("*=", Punct::StarEq), + ("%=", Punct::PercentEq), + ("&=", Punct::AmpEq), + ("|=", Punct::PipeEq), + ("^=", Punct::CaretEq), + ("/=", Punct::SlashEq), + ("{", Punct::LBrace), + ("}", Punct::RBrace), + ("(", Punct::LParen), + (")", Punct::RParen), + ("[", Punct::LBracket), + ("]", Punct::RBracket), + (".", Punct::Dot), + (";", Punct::Semicolon), + (",", Punct::Comma), + ("<", Punct::Lt), + (">", Punct::Gt), + ("+", Punct::Plus), + ("-", Punct::Minus), + ("*", Punct::Star), + ("%", Punct::Percent), + ("&", Punct::Amp), + ("|", Punct::Pipe), + ("^", Punct::Caret), + ("!", Punct::Bang), + ("~", Punct::Tilde), + ("?", Punct::Question), + (":", Punct::Colon), + ("=", Punct::Eq), + ("/", Punct::Slash), + ]; + + pub fn as_str(self) -> &'static str { + Punct::ALL.iter().find(|(_, p)| *p == self).map_or("", |(s, _)| s) + } +} + +/// A preprocessor line's directive: `#include`, `#includepath`, `#script`, `#strict`, `#target` +/// and `#targetengine`, as the public JavaScript Tools Guide lists them +/// (). +/// Only the names are taken from it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DirectiveName { + Include, + IncludePath, + Script, + Strict, + Target, + TargetEngine, +} + +impl DirectiveName { + /// The directive named `name`, compared without case. + pub fn from_name(name: &str) -> Option { + const NAMES: [(&str, DirectiveName); 6] = [ + ("include", DirectiveName::Include), + ("includepath", DirectiveName::IncludePath), + ("script", DirectiveName::Script), + ("strict", DirectiveName::Strict), + ("target", DirectiveName::Target), + ("targetengine", DirectiveName::TargetEngine), + ]; + NAMES.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, d)| *d) + } + + pub fn as_str(self) -> &'static str { + match self { + DirectiveName::Include => "include", + DirectiveName::IncludePath => "includepath", + DirectiveName::Script => "script", + DirectiveName::Strict => "strict", + DirectiveName::Target => "target", + DirectiveName::TargetEngine => "targetengine", + } + } +} + +/// A preprocessor line: `#name value` at the start of a line, or a `//@name value` comment there +/// that names a directive. It is read and kept, never run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Directive { + /// The directive, when the name is one of the six (`#name` lines with another name are kept + /// with `None`). + pub known: Option, + /// The name as written. + pub name: String, + /// The text after the name, with blanks around it removed. + pub value: String, + /// `#` or `//@`. + pub form: &'static str, + pub span: Span, + pub line: u32, +} + +/// What a token is. +#[derive(Clone, Debug, PartialEq)] +pub enum TokenKind { + /// The end of the source. + Eof, + /// An identifier, the strict-mode reserved words included. `escaped` tokens held a `\u` escape. + Ident(String), + Keyword(Keyword), + Punct(Punct), + /// A numeric literal's value. + Number(f64), + /// A string literal's value in UTF-16 code units. + String(Vec), + /// A regular expression literal: its pattern and flags as written. + RegExp { + pattern: Vec, + flags: String, + }, + /// A preprocessor line. + Directive(Directive), +} + +/// A token, with where it is and the facts the grammar's rules need about it. +#[derive(Clone, Debug, PartialEq)] +pub struct Token { + pub kind: TokenKind, + pub span: Span, + /// The line (from 1) and column (from 1, in UTF-16 code units) the token starts at. + pub line: u32, + pub column: u32, + /// A line terminator comes between the previous token and this one (automatic semicolon + /// insertion and the restricted productions use it). + pub newline_before: bool, + /// An identifier written with a `\u` escape. + pub escaped: bool, + /// A legacy octal numeric literal (`017`, `08`) or a string with an octal escape (`"\1"`), which + /// strict mode code does not allow. + pub legacy_octal: bool, +} + +impl Token { + pub fn is_punct(&self, p: Punct) -> bool { + self.kind == TokenKind::Punct(p) + } + + pub fn is_keyword(&self, k: Keyword) -> bool { + self.kind == TokenKind::Keyword(k) + } + + /// The identifier's name, for an identifier token. + pub fn ident(&self) -> Option<&str> { + match &self.kind { + TokenKind::Ident(name) => Some(name), + _ => None, + } + } + + /// The name this token gives when it stands where any identifier name may, as a property's + /// name after `.` or in an object literal: identifiers, keywords and the literals `null`, + /// `true` and `false`. + pub fn identifier_name(&self) -> Option<&str> { + match &self.kind { + TokenKind::Ident(name) => Some(name), + TokenKind::Keyword(k) => Some(k.as_str()), + _ => None, + } + } +} diff --git a/crates/js/tests/test262.rs b/crates/js/tests/test262.rs new file mode 100644 index 000000000..c32045507 --- /dev/null +++ b/crates/js/tests/test262.rs @@ -0,0 +1,178 @@ +//! The ES5 parse subset of test262, the ECMAScript conformance suite, when `cargo xtask test262` +//! has fetched it (it sets `VECTORCRAFT_TEST262_DIR`; without it the test looks in +//! `corpus/test262/` and skips when the suite is absent). +//! +//! The subset is every test under `test/language/` that names the ES5 section it was written for +//! (`es5id:`) and has no `features:` tag (the tags name syntax and built-ins later than ES5) and no +//! `module` flag. The other tests without a feature tag (written for later editions, many of them +//! using later syntax) run too, and their counts are printed apart. A test whose metadata says +//! `negative: phase: parse` must give a `SyntaxError`; every other test must parse. Each test runs +//! as sloppy and as strict mode code (with `"use strict";` before it, as the suite's +//! `INTERPRETING.md` says), unless its `onlyStrict`, `noStrict` or `raw` flag says otherwise. The +//! test prints the counts and writes each failure to `failures.txt` beside the suite; it fails only +//! when the suite cannot be read. + +use std::path::{Path, PathBuf}; + +use vectorcraft_js::{Options, parse}; + +/// A test file's metadata, from its `/*--- … ---*/` block. +#[derive(Default)] +struct Meta { + features: Vec, + flags: Vec, + negative_phase: Option, + /// The test names the ES5 section it was written for (`es5id:`). + es5: bool, +} + +fn suite_dir() -> Option { + if let Ok(dir) = std::env::var("VECTORCRAFT_TEST262_DIR") { + return Some(PathBuf::from(dir)); + } + let corpus = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/test262"); + let mut dirs: Vec = std::fs::read_dir(corpus).ok()?.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.is_dir()).collect(); + dirs.sort(); + dirs.pop() +} + +/// The `.js` files under `dir`, sorted, with the `_FIXTURE` helper files left out. +fn test_files(dir: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&d) else { continue }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|e| e == "js") && !path.to_string_lossy().contains("_FIXTURE") { + out.push(path); + } + } + } + out.sort(); + out +} + +/// A YAML flow list (`[a, b]`) or the block list (`- a` lines) after a key. +fn list(value: &str, lines: &mut std::iter::Peekable>) -> Vec { + let value = value.trim(); + if let Some(inner) = value.strip_prefix('[').and_then(|v| v.strip_suffix(']')) { + return inner.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect(); + } + let mut out = Vec::new(); + while let Some(item) = lines.peek().and_then(|l| l.trim().strip_prefix("- ")) { + out.push(item.trim().to_string()); + lines.next(); + } + out +} + +fn meta(source: &str) -> Meta { + let mut meta = Meta::default(); + let Some(block) = source.split_once("/*---").and_then(|(_, rest)| rest.split_once("---*/")).map(|(block, _)| block) else { + return meta; + }; + let mut lines = block.lines().peekable(); + let mut in_negative = false; + while let Some(line) = lines.next() { + let indented = line.starts_with(' '); + let trimmed = line.trim(); + if !indented { + in_negative = false; + } + if let Some(v) = trimmed.strip_prefix("features:").filter(|_| !indented) { + meta.features = list(v, &mut lines); + } else if let Some(v) = trimmed.strip_prefix("flags:").filter(|_| !indented) { + meta.flags = list(v, &mut lines); + } else if trimmed.starts_with("es5id:") && !indented { + meta.es5 = true; + } else if trimmed == "negative:" && !indented { + in_negative = true; + } else if let Some(v) = trimmed.strip_prefix("phase:").filter(|_| in_negative) { + meta.negative_phase = Some(v.trim().to_string()); + } + } + meta +} + +#[test] +fn es5_parse_subset() { + let Some(dir) = suite_dir() else { + println!("test262: SKIPPED (no suite; run cargo xtask test262)"); + return; + }; + let language = dir.join("test").join("language"); + assert!(language.is_dir(), "{} is not a test262 checkout", dir.display()); + let files = test_files(&language); + assert!(!files.is_empty(), "no tests under {}", language.display()); + // VECTORCRAFT_TEST262_VERBOSE=1 prints each test before it is parsed. + let verbose = std::env::var_os("VECTORCRAFT_TEST262_VERBOSE").is_some(); + + let (mut positive, mut negative, mut with_features, mut modules) = (0usize, 0usize, 0usize, 0usize); + let (mut passed, mut failed) = (0usize, 0usize); + // The tests without a feature tag that name no ES5 section: files, runs passed and failed. + let (mut later, mut later_passed, mut later_failed) = (0usize, 0usize, 0usize); + let mut failures = String::new(); + let mut later_failures = String::new(); + for path in &files { + let source = String::from_utf8_lossy(&std::fs::read(path).unwrap()).into_owned(); + let meta = meta(&source); + if !meta.features.is_empty() { + with_features += 1; + continue; + } + if meta.flags.iter().any(|f| f == "module") || meta.negative_phase.as_deref() == Some("resolution") { + modules += 1; + continue; + } + let expect_error = meta.negative_phase.as_deref() == Some("parse"); + if !meta.es5 { + later += 1; + } else if expect_error { + negative += 1; + } else { + positive += 1; + } + let has = |flag: &str| meta.flags.iter().any(|f| f == flag); + let modes: &[bool] = if has("raw") || has("noStrict") { + &[false] + } else if has("onlyStrict") { + &[true] + } else { + &[false, true] + }; + for &strict in modes { + if verbose { + eprintln!("{} ({})", path.display(), if strict { "strict" } else { "sloppy" }); + } + let text = if strict { format!("\"use strict\";\n{source}") } else { source.clone() }; + let result = parse(&text, &Options::default()); + let (pass, fail, list) = + if meta.es5 { (&mut passed, &mut failed, &mut failures) } else { (&mut later_passed, &mut later_failed, &mut later_failures) }; + if result.is_err() == expect_error { + *pass += 1; + } else { + *fail += 1; + let rel = path.strip_prefix(&dir).unwrap_or(path).display().to_string().replace('\\', "/"); + let mode = if strict { "strict" } else { "sloppy" }; + let outcome = match &result { + Ok(_) => "parsed; a SyntaxError was expected".to_string(), + Err(e) => e.to_string(), + }; + list.push_str(&format!("{rel} ({mode}): {outcome}\n")); + } + } + } + let report = dir.join("failures.txt"); + std::fs::write(&report, format!("# ES5 subset\n{failures}\n# Other tests without a feature tag\n{later_failures}")).unwrap(); + println!( + "test262 ES5 parse subset ({}): {} tests ({positive} that must parse, {negative} that must give a SyntaxError), \ + {passed} runs passed, {failed} failed. Other tests without a feature tag: {later} tests, {later_passed} runs passed, \ + {later_failed} failed. Skipped {with_features} tests with feature tags and {modules} module tests. Failures: {}", + dir.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), + positive + negative, + report.display() + ); +} diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 723fc673a..57540ba68 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -662,6 +662,7 @@ Moved from the ROADMAP's "Shipped so far" on 2026-10-10. Update it when a featur - MCP protocol: five prompts (`prompts/list` / `prompts/get`: poster, icon set, recolour, trace-and-style, export set), `completion/complete` for prompt arguments and template variables read from the live catalogues (formats, effect ids, command ids, trace presets, swatch and object ids), four resource templates (`vectorcraft://object/{id}`, `//command/{id}`, `//effect/{id}`, `//swatch/{name}`) so an agent reads one thing instead of the whole document, and `logging/setLevel` with `notifications/message`. `window.panel` ids are case-insensitive (display labels work too), `vectorcraft://object/{id}` reads through `document.node {summary: true}` (the same reply bytes from one lookup instead of summarizing the whole document: a leaf read on a 2,000-object document drops from ~20 ms to ~0.06 ms), large containers slice with `document.node {depth?, childLimit?}` (truncated levels report `childCount`), `document.inspect` and the `inspect_document` tool slice the same way so a depth-0 read is the layer skeleton (~9.7 KB instead of ~2 MB on a 2,000-object document), `document.find` locates nodes by name/kind/text with ancestor paths, and the CLI sweeps the protocol surface over stdio against the compiled binary. `docs/mcp.md` has a Protocol section covering the capability table and what is deliberately absent (subscriptions, progress, cancellation, sampling, elicitation, Streamable HTTP — all of which need the transport to interleave reads). - Headless CLI (`vectorcraft-cli run`, `convert`, `info`, `bench`, `perf`, `mcp`). A `run` step, like a `command_batch` step over MCP, can use an earlier step's result (`"$1.id"`, #785); `run --commands FILE` (or `-` for standard input) reads the steps from a JSON array or JSON Lines, past the Windows command-line limit (#3147, `apps/vectorcraft-cli/tests/it/cli.rs` `run_reads_*`, `run_refuses_malformed_commands_naming_the_step`). - File › Scripts › Check Script… reads a script written for Illustrator (`.jsx`, `.jsxinc` or a compiled `.jsxbin`) without running any of it and reports what it needs (the document model, files, dialogs, menu commands, features only Illustrator has), what it would do outside the document, the menu commands and document-model members it uses, its directives and included files, and the problems that would stop it loading (`script.check`, which also reads `.js` files); File › Open and drag-and-drop show the report for `.jsx`, `.jsxinc` and `.jsxbin` files. + - `vectorcraft-js` parses JavaScript as ECMAScript 5.1 defines it, with the legacy octal forms, block functions and lenient regular expressions of Annex B, into a syntax tree with source spans; it reads the `#target`, `#include` and other preprocessor lines as directives without running them, and a script that breaks the grammar or an early-error rule gives a `SyntaxError` with its line and column. Nesting past the parser's depth limit (`Options::max_depth`, 200 by default) gives a `SyntaxError` instead of exhausting the stack. `cargo xtask test262` fetches the conformance suite at a pinned commit and parses its ES5 tests: 5,411 of 5,418 runs pass (the 7 others use `let` and `const`). Nothing runs scripts yet. Tests: `crates/js/src/tests.rs`, `crates/engine/tests/it/js_lexer_vs_scriptcheck.rs`. - **UI:** Illustrator 2026 layout restyled to measured values: - Medium Dark theme, categorized and Advanced toolbars (a tool group's flyout opens on a right press, a long press or a press on its corner triangle), 35 pt document tabs, 33 pt panel tabs. - Hint bar, contextual task bar, 19 dock panels with ≡ menus. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 3117a7af1..6b8bda8d1 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -63,6 +63,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("plugins", Class::Layer(2)), ("affinity", Class::Standalone), ("scriptcheck", Class::Standalone), + ("js", Class::Standalone), ]; /// Explicit orderings *within* a layer (earlier may be used by later). diff --git a/xtask/src/main.rs b/xtask/src/main.rs index de6ff5815..334910732 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -14,6 +14,7 @@ mod pdf_deps; mod pngsuite; mod sha256; mod stats; +mod test262; mod version; mod web_fonts; @@ -41,6 +42,8 @@ commands: ico ... pack PNGs into a Windows .ico (used by packaging/icons.sh) stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) + test262 fetch the pinned ECMAScript conformance suite into corpus/test262 (sha256 + verified) and parse its ES5 subset with vectorcraft-js, printing the counts web-fonts copy the craft-fonts faces in crates/text/web-fonts.txt into the web site (the Trunk post_build hook; a no-op without CRAFT_FONTS_DIR) "; @@ -64,6 +67,7 @@ fn main() -> ExitCode { Some("ico") => ico::run(&rest), Some("version") => version::run(&root(), &rest), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), + Some("test262") => test262::run_suite(), Some("web-fonts") => web_fonts::run(&root()), Some("-h" | "--help" | "help") | None => { print!("{USAGE}"); @@ -333,6 +337,8 @@ Tests that use a corpus skip cleanly when it is absent. corpus/raw/ raw.pixls.us samples, CC0 (optional) corpus/affinity/ public Affinity documents (CC0/MIT/Apache-2.0) at pinned commits, sha256 verified: oracles for the Affinity reader. Fetch: cargo xtask corpus --affinity + corpus/test262/ the ECMAScript conformance suite (BSD-3-Clause) at a pinned commit, sha256 + verified: the JavaScript parser's ES5 subset. Fetch and run: cargo xtask test262 ", corpus.display() ); diff --git a/xtask/src/test262.rs b/xtask/src/test262.rs new file mode 100644 index 000000000..ab390538b --- /dev/null +++ b/xtask/src/test262.rs @@ -0,0 +1,73 @@ +//! `cargo xtask test262`: the official ECMAScript conformance suite (https://github.com/tc39/test262, +//! BSD-3-Clause) at a pinned commit, fetched as GitHub's archive of that commit, checked against +//! its SHA-256 and size, and unpacked into `corpus/test262/` (gitignored, never committed). It then +//! runs `vectorcraft-js`'s `test262` test, which parses the ES5 parse subset and prints its counts. +//! +//! Moving the pin: change [`COMMIT`], run the command once with the new commit to see the +//! archive's SHA-256 in the error, check the archive, and update [`SHA256`]. + +use std::path::Path; +use std::process::Command; + +use crate::{cargo, root, run, sha256}; + +/// The tc39/test262 commit the suite is read at. +const COMMIT: &str = "d53ebc8b1931d9f1541e43d59650776e4b766248"; + +/// The SHA-256 of GitHub's `tar.gz` archive of [`COMMIT`]. +const SHA256: &str = "6db8566044d1a3f5a68aac87806e09b2fe5d21f5869f3a74cfd8b7236ebaf5c8"; + +/// The largest archive accepted (the pinned one is 9.6 MB). +const MAX_BYTES: u64 = 64 << 20; + +pub fn run_suite() -> Result<(), String> { + let dir = fetch()?; + let mut c = cargo(); + c.env("VECTORCRAFT_TEST262_DIR", &dir).args(["test", "-p", "vectorcraft-js", "--release", "--test", "test262", "--", "--nocapture"]); + run(c, &format!("cargo test -p vectorcraft-js --release --test test262 -- --nocapture (VECTORCRAFT_TEST262_DIR={})", dir.display())) +} + +/// Fetches and unpacks the suite unless it is there already, and returns its folder. +fn fetch() -> Result { + let base = root().join("corpus").join("test262"); + let dir = base.join(format!("test262-{COMMIT}")); + let marker = dir.join(".vectorcraft-sha256"); + if std::fs::read_to_string(&marker).is_ok_and(|s| s.trim() == SHA256) { + println!("test262 {COMMIT} is in {}", dir.display()); + return Ok(dir); + } + std::fs::create_dir_all(&base).map_err(|e| format!("create {}: {e}", base.display()))?; + let archive = base.join(format!("test262-{COMMIT}.tar.gz")); + if !verified(&archive)? { + let url = format!("https://codeload.github.com/tc39/test262/tar.gz/{COMMIT}"); + let mut curl = Command::new("curl"); + curl.args(["-fsSL", "--retry", "3", "-A", "VectorCraft-dev", "--max-filesize", &MAX_BYTES.to_string(), "-o"]).arg(&archive).arg(&url); + run(curl, &format!("curl {url}"))?; + if !verified(&archive)? { + let got = std::fs::read(&archive).map(|b| sha256::hex(&b)).unwrap_or_default(); + let _ = std::fs::remove_file(&archive); + return Err(format!("test262 archive: sha256 {got} does not match the pinned {SHA256}")); + } + } + if dir.exists() { + std::fs::remove_dir_all(&dir).map_err(|e| format!("remove {}: {e}", dir.display()))?; + } + let mut tar = Command::new("tar"); + tar.arg("-xzf").arg(&archive).arg("-C").arg(&base); + run(tar, &format!("tar -xzf {}", archive.display()))?; + if !dir.join("test").join("language").is_dir() { + return Err(format!("{} has no test/language folder after unpacking", dir.display())); + } + std::fs::write(&marker, SHA256).map_err(|e| format!("{}: {e}", marker.display()))?; + println!("test262 {COMMIT} unpacked into {}", dir.display()); + Ok(dir) +} + +/// The archive is there, within the size limit, with the pinned SHA-256. +fn verified(path: &Path) -> Result { + let Ok(meta) = std::fs::metadata(path) else { return Ok(false) }; + if meta.len() > MAX_BYTES { + return Err(format!("{} is {} bytes, over the {MAX_BYTES}-byte limit", path.display(), meta.len())); + } + Ok(std::fs::read(path).is_ok_and(|b| sha256::hex(&b) == SHA256)) +}