1777 lines
54 KiB
Rust
1777 lines
54 KiB
Rust
//! Main module defining the lexer and parser.
|
|
|
|
use crate::engine::{
|
|
KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL, KEYWORD_FN_PTR_CURRY,
|
|
KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
|
|
};
|
|
use crate::stdlib::{
|
|
borrow::Cow,
|
|
boxed::Box,
|
|
char, fmt, format,
|
|
iter::Peekable,
|
|
str::{Chars, FromStr},
|
|
string::{String, ToString},
|
|
};
|
|
use crate::{Engine, LexError, StaticVec, INT};
|
|
|
|
#[cfg(not(feature = "no_float"))]
|
|
use crate::FLOAT;
|
|
|
|
type LERR = LexError;
|
|
|
|
pub type TokenStream<'a, 't> = Peekable<TokenIterator<'a, 't>>;
|
|
|
|
/// A location (line number + character position) in the input script.
|
|
///
|
|
/// # Limitations
|
|
///
|
|
/// In order to keep footprint small, both line number and character position have 16-bit resolution,
|
|
/// meaning they go up to a maximum of 65,535 lines and 65,535 characters per line.
|
|
///
|
|
/// Advancing beyond the maximum line length or maximum number of lines is not an error but has no effect.
|
|
#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy)]
|
|
pub struct Position {
|
|
/// Line number - 0 = none
|
|
line: u16,
|
|
/// Character position - 0 = BOL
|
|
pos: u16,
|
|
}
|
|
|
|
impl Position {
|
|
/// A [`Position`] representing no position.
|
|
pub const NONE: Self = Self { line: 0, pos: 0 };
|
|
/// A [`Position`] representing the first position.
|
|
pub const START: Self = Self { line: 1, pos: 0 };
|
|
|
|
/// Create a new [`Position`].
|
|
///
|
|
/// `line` must not be zero.
|
|
/// If [`Position`] is zero, then it is at the beginning of a line.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Panics if `line` is zero.
|
|
#[inline(always)]
|
|
pub fn new(line: u16, position: u16) -> Self {
|
|
assert!(line != 0, "line cannot be zero");
|
|
|
|
Self {
|
|
line,
|
|
pos: position,
|
|
}
|
|
}
|
|
/// Get the line number (1-based), or [`None`] if there is no position.
|
|
#[inline(always)]
|
|
pub fn line(self) -> Option<usize> {
|
|
if self.is_none() {
|
|
None
|
|
} else {
|
|
Some(self.line as usize)
|
|
}
|
|
}
|
|
/// Get the character position (1-based), or [`None`] if at beginning of a line.
|
|
#[inline(always)]
|
|
pub fn position(self) -> Option<usize> {
|
|
if self.is_none() || self.pos == 0 {
|
|
None
|
|
} else {
|
|
Some(self.pos as usize)
|
|
}
|
|
}
|
|
/// Advance by one character position.
|
|
#[inline(always)]
|
|
pub(crate) fn advance(&mut self) {
|
|
assert!(!self.is_none(), "cannot advance Position::none");
|
|
|
|
// Advance up to maximum position
|
|
if self.pos < u16::MAX {
|
|
self.pos += 1;
|
|
}
|
|
}
|
|
/// Go backwards by one character position.
|
|
///
|
|
/// # Panics
|
|
///
|
|
/// Panics if already at beginning of a line - cannot rewind to a previous line.
|
|
#[inline(always)]
|
|
pub(crate) fn rewind(&mut self) {
|
|
assert!(!self.is_none(), "cannot rewind Position::none");
|
|
assert!(self.pos > 0, "cannot rewind at position 0");
|
|
self.pos -= 1;
|
|
}
|
|
/// Advance to the next line.
|
|
#[inline(always)]
|
|
pub(crate) fn new_line(&mut self) {
|
|
assert!(!self.is_none(), "cannot advance Position::none");
|
|
|
|
// Advance up to maximum position
|
|
if self.line < u16::MAX {
|
|
self.line += 1;
|
|
self.pos = 0;
|
|
}
|
|
}
|
|
/// Is this [`Position`] at the beginning of a line?
|
|
#[inline(always)]
|
|
pub fn is_beginning_of_line(self) -> bool {
|
|
self.line == 0 && !self.is_none()
|
|
}
|
|
/// Is there no [`Position`]?
|
|
#[inline(always)]
|
|
pub fn is_none(self) -> bool {
|
|
self == Self::NONE
|
|
}
|
|
}
|
|
|
|
impl Default for Position {
|
|
#[inline(always)]
|
|
fn default() -> Self {
|
|
Self::START
|
|
}
|
|
}
|
|
|
|
impl fmt::Display for Position {
|
|
#[inline(always)]
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
if self.is_none() {
|
|
write!(f, "none")
|
|
} else {
|
|
write!(f, "line {}, position {}", self.line, self.pos)
|
|
}
|
|
}
|
|
}
|
|
|
|
impl fmt::Debug for Position {
|
|
#[inline(always)]
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
write!(f, "{}:{}", self.line, self.pos)
|
|
}
|
|
}
|
|
|
|
/// _(INTERNALS)_ A Rhai language token.
|
|
/// Exported under the `internals` feature only.
|
|
///
|
|
/// ## WARNING
|
|
///
|
|
/// This type is volatile and may change.
|
|
#[derive(Debug, PartialEq, Clone)]
|
|
pub enum Token {
|
|
/// An `INT` constant.
|
|
IntegerConstant(INT),
|
|
/// A `FLOAT` constant.
|
|
///
|
|
/// Reserved under the `no_float` feature.
|
|
#[cfg(not(feature = "no_float"))]
|
|
FloatConstant(FLOAT),
|
|
/// An identifier.
|
|
Identifier(String),
|
|
/// A character constant.
|
|
CharConstant(char),
|
|
/// A string constant.
|
|
StringConstant(String),
|
|
/// `{`
|
|
LeftBrace,
|
|
/// `}`
|
|
RightBrace,
|
|
/// `(`
|
|
LeftParen,
|
|
/// `)`
|
|
RightParen,
|
|
/// `[`
|
|
LeftBracket,
|
|
/// `]`
|
|
RightBracket,
|
|
/// `+`
|
|
Plus,
|
|
/// `+` (unary)
|
|
UnaryPlus,
|
|
/// `-`
|
|
Minus,
|
|
/// `-` (unary)
|
|
UnaryMinus,
|
|
/// `*`
|
|
Multiply,
|
|
/// `/`
|
|
Divide,
|
|
/// `%`
|
|
Modulo,
|
|
/// `~`
|
|
PowerOf,
|
|
/// `<<`
|
|
LeftShift,
|
|
/// `>>`
|
|
RightShift,
|
|
/// `;`
|
|
SemiColon,
|
|
/// `:`
|
|
Colon,
|
|
/// `::`
|
|
DoubleColon,
|
|
/// `=>`
|
|
DoubleArrow,
|
|
/// `_`
|
|
Underscore,
|
|
/// `,`
|
|
Comma,
|
|
/// `.`
|
|
Period,
|
|
/// `#{`
|
|
MapStart,
|
|
/// `=`
|
|
Equals,
|
|
/// `true`
|
|
True,
|
|
/// `false`
|
|
False,
|
|
/// `let`
|
|
Let,
|
|
/// `const`
|
|
Const,
|
|
/// `if`
|
|
If,
|
|
/// `else`
|
|
Else,
|
|
/// `switch`
|
|
Switch,
|
|
/// `do`
|
|
Do,
|
|
/// `while`
|
|
While,
|
|
/// `until`
|
|
Until,
|
|
/// `loop`
|
|
Loop,
|
|
/// `for`
|
|
For,
|
|
/// `in`
|
|
In,
|
|
/// `<`
|
|
LessThan,
|
|
/// `>`
|
|
GreaterThan,
|
|
/// `<=`
|
|
LessThanEqualsTo,
|
|
/// `>=`
|
|
GreaterThanEqualsTo,
|
|
/// `==`
|
|
EqualsTo,
|
|
/// `!=`
|
|
NotEqualsTo,
|
|
/// `!`
|
|
Bang,
|
|
/// `|`
|
|
Pipe,
|
|
/// `||`
|
|
Or,
|
|
/// `^`
|
|
XOr,
|
|
/// `&`
|
|
Ampersand,
|
|
/// `&&`
|
|
And,
|
|
/// `fn`
|
|
///
|
|
/// Reserved under the `no_function` feature.
|
|
#[cfg(not(feature = "no_function"))]
|
|
Fn,
|
|
/// `continue`
|
|
Continue,
|
|
/// `break`
|
|
Break,
|
|
/// `return`
|
|
Return,
|
|
/// `throw`
|
|
Throw,
|
|
/// `try`
|
|
Try,
|
|
/// `catch`
|
|
Catch,
|
|
/// `+=`
|
|
PlusAssign,
|
|
/// `-=`
|
|
MinusAssign,
|
|
/// `*=`
|
|
MultiplyAssign,
|
|
/// `/=`
|
|
DivideAssign,
|
|
/// `<<=`
|
|
LeftShiftAssign,
|
|
/// `>>=`
|
|
RightShiftAssign,
|
|
/// `&=`
|
|
AndAssign,
|
|
/// `|=`
|
|
OrAssign,
|
|
/// `^=`
|
|
XOrAssign,
|
|
/// `%=`
|
|
ModuloAssign,
|
|
/// `~=`
|
|
PowerOfAssign,
|
|
/// `private`
|
|
///
|
|
/// Reserved under the `no_function` feature.
|
|
#[cfg(not(feature = "no_function"))]
|
|
Private,
|
|
/// `import`
|
|
///
|
|
/// Reserved under the `no_module` feature.
|
|
#[cfg(not(feature = "no_module"))]
|
|
Import,
|
|
/// `export`
|
|
///
|
|
/// Reserved under the `no_module` feature.
|
|
#[cfg(not(feature = "no_module"))]
|
|
Export,
|
|
/// `as`
|
|
///
|
|
/// Reserved under the `no_module` feature.
|
|
#[cfg(not(feature = "no_module"))]
|
|
As,
|
|
/// A lexer error.
|
|
LexError(LexError),
|
|
/// A comment block.
|
|
Comment(String),
|
|
/// A reserved symbol.
|
|
Reserved(String),
|
|
/// A custom keyword.
|
|
Custom(String),
|
|
/// End of the input stream.
|
|
EOF,
|
|
}
|
|
|
|
impl Token {
|
|
/// Get the syntax of the token.
|
|
pub fn syntax(&self) -> Cow<'static, str> {
|
|
use Token::*;
|
|
|
|
match self {
|
|
IntegerConstant(i) => i.to_string().into(),
|
|
#[cfg(not(feature = "no_float"))]
|
|
FloatConstant(f) => f.to_string().into(),
|
|
StringConstant(_) => "string".into(),
|
|
CharConstant(c) => c.to_string().into(),
|
|
Identifier(s) => s.clone().into(),
|
|
Reserved(s) => s.clone().into(),
|
|
Custom(s) => s.clone().into(),
|
|
LexError(err) => err.to_string().into(),
|
|
|
|
token => match token {
|
|
LeftBrace => "{",
|
|
RightBrace => "}",
|
|
LeftParen => "(",
|
|
RightParen => ")",
|
|
LeftBracket => "[",
|
|
RightBracket => "]",
|
|
Plus => "+",
|
|
UnaryPlus => "+",
|
|
Minus => "-",
|
|
UnaryMinus => "-",
|
|
Multiply => "*",
|
|
Divide => "/",
|
|
SemiColon => ";",
|
|
Colon => ":",
|
|
DoubleColon => "::",
|
|
DoubleArrow => "=>",
|
|
Underscore => "_",
|
|
Comma => ",",
|
|
Period => ".",
|
|
MapStart => "#{",
|
|
Equals => "=",
|
|
True => "true",
|
|
False => "false",
|
|
Let => "let",
|
|
Const => "const",
|
|
If => "if",
|
|
Else => "else",
|
|
Switch => "switch",
|
|
Do => "do",
|
|
While => "while",
|
|
Until => "until",
|
|
Loop => "loop",
|
|
For => "for",
|
|
In => "in",
|
|
LessThan => "<",
|
|
GreaterThan => ">",
|
|
Bang => "!",
|
|
LessThanEqualsTo => "<=",
|
|
GreaterThanEqualsTo => ">=",
|
|
EqualsTo => "==",
|
|
NotEqualsTo => "!=",
|
|
Pipe => "|",
|
|
Or => "||",
|
|
Ampersand => "&",
|
|
And => "&&",
|
|
Continue => "continue",
|
|
Break => "break",
|
|
Return => "return",
|
|
Throw => "throw",
|
|
Try => "try",
|
|
Catch => "catch",
|
|
PlusAssign => "+=",
|
|
MinusAssign => "-=",
|
|
MultiplyAssign => "*=",
|
|
DivideAssign => "/=",
|
|
LeftShiftAssign => "<<=",
|
|
RightShiftAssign => ">>=",
|
|
AndAssign => "&=",
|
|
OrAssign => "|=",
|
|
XOrAssign => "^=",
|
|
LeftShift => "<<",
|
|
RightShift => ">>",
|
|
XOr => "^",
|
|
Modulo => "%",
|
|
ModuloAssign => "%=",
|
|
PowerOf => "~",
|
|
PowerOfAssign => "~=",
|
|
|
|
#[cfg(not(feature = "no_function"))]
|
|
Fn => "fn",
|
|
#[cfg(not(feature = "no_function"))]
|
|
Private => "private",
|
|
|
|
#[cfg(not(feature = "no_module"))]
|
|
Import => "import",
|
|
#[cfg(not(feature = "no_module"))]
|
|
Export => "export",
|
|
#[cfg(not(feature = "no_module"))]
|
|
As => "as",
|
|
EOF => "{EOF}",
|
|
_ => unreachable!("operator should be match in outer scope"),
|
|
}
|
|
.into(),
|
|
}
|
|
}
|
|
|
|
/// Reverse lookup a token from a piece of syntax.
|
|
pub fn lookup_from_syntax(syntax: &str) -> Option<Self> {
|
|
use Token::*;
|
|
|
|
Some(match syntax {
|
|
"{" => LeftBrace,
|
|
"}" => RightBrace,
|
|
"(" => LeftParen,
|
|
")" => RightParen,
|
|
"[" => LeftBracket,
|
|
"]" => RightBracket,
|
|
"+" => Plus,
|
|
"-" => Minus,
|
|
"*" => Multiply,
|
|
"/" => Divide,
|
|
";" => SemiColon,
|
|
":" => Colon,
|
|
"::" => DoubleColon,
|
|
"=>" => DoubleArrow,
|
|
"_" => Underscore,
|
|
"," => Comma,
|
|
"." => Period,
|
|
"#{" => MapStart,
|
|
"=" => Equals,
|
|
"true" => True,
|
|
"false" => False,
|
|
"let" => Let,
|
|
"const" => Const,
|
|
"if" => If,
|
|
"else" => Else,
|
|
"switch" => Switch,
|
|
"do" => Do,
|
|
"while" => While,
|
|
"until" => Until,
|
|
"loop" => Loop,
|
|
"for" => For,
|
|
"in" => In,
|
|
"<" => LessThan,
|
|
">" => GreaterThan,
|
|
"!" => Bang,
|
|
"<=" => LessThanEqualsTo,
|
|
">=" => GreaterThanEqualsTo,
|
|
"==" => EqualsTo,
|
|
"!=" => NotEqualsTo,
|
|
"|" => Pipe,
|
|
"||" => Or,
|
|
"&" => Ampersand,
|
|
"&&" => And,
|
|
"continue" => Continue,
|
|
"break" => Break,
|
|
"return" => Return,
|
|
"throw" => Throw,
|
|
"try" => Try,
|
|
"catch" => Catch,
|
|
"+=" => PlusAssign,
|
|
"-=" => MinusAssign,
|
|
"*=" => MultiplyAssign,
|
|
"/=" => DivideAssign,
|
|
"<<=" => LeftShiftAssign,
|
|
">>=" => RightShiftAssign,
|
|
"&=" => AndAssign,
|
|
"|=" => OrAssign,
|
|
"^=" => XOrAssign,
|
|
"<<" => LeftShift,
|
|
">>" => RightShift,
|
|
"^" => XOr,
|
|
"%" => Modulo,
|
|
"%=" => ModuloAssign,
|
|
"~" => PowerOf,
|
|
"~=" => PowerOfAssign,
|
|
|
|
#[cfg(not(feature = "no_function"))]
|
|
"fn" => Fn,
|
|
#[cfg(not(feature = "no_function"))]
|
|
"private" => Private,
|
|
|
|
#[cfg(not(feature = "no_module"))]
|
|
"import" => Import,
|
|
#[cfg(not(feature = "no_module"))]
|
|
"export" => Export,
|
|
#[cfg(not(feature = "no_module"))]
|
|
"as" => As,
|
|
|
|
#[cfg(feature = "no_function")]
|
|
"fn" | "private" => Reserved(syntax.into()),
|
|
|
|
#[cfg(feature = "no_module")]
|
|
"import" | "export" | "as" => Reserved(syntax.into()),
|
|
|
|
"===" | "!==" | "->" | "<-" | ":=" | "::<" | "(*" | "*)" | "#" | "public" | "new"
|
|
| "use" | "module" | "package" | "var" | "static" | "begin" | "end" | "shared"
|
|
| "with" | "each" | "then" | "goto" | "unless" | "exit" | "match" | "case"
|
|
| "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync"
|
|
| "async" | "await" | "yield" => Reserved(syntax.into()),
|
|
|
|
KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR
|
|
| KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY | KEYWORD_THIS => Reserved(syntax.into()),
|
|
|
|
_ => return None,
|
|
})
|
|
}
|
|
|
|
// Is this token [`EOF`][Token::EOF]?
|
|
#[inline(always)]
|
|
pub fn is_eof(&self) -> bool {
|
|
use Token::*;
|
|
|
|
match self {
|
|
EOF => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
// If another operator is after these, it's probably an unary operator
|
|
// (not sure about `fn` name).
|
|
pub fn is_next_unary(&self) -> bool {
|
|
use Token::*;
|
|
|
|
match self {
|
|
LexError(_) |
|
|
LeftBrace | // {+expr} - is unary
|
|
// RightBrace | {expr} - expr not unary & is closing
|
|
LeftParen | // (-expr) - is unary
|
|
// RightParen | (expr) - expr not unary & is closing
|
|
LeftBracket | // [-expr] - is unary
|
|
// RightBracket | [expr] - expr not unary & is closing
|
|
Plus |
|
|
UnaryPlus |
|
|
Minus |
|
|
UnaryMinus |
|
|
Multiply |
|
|
Divide |
|
|
Comma |
|
|
Period |
|
|
Equals |
|
|
LessThan |
|
|
GreaterThan |
|
|
Bang |
|
|
LessThanEqualsTo |
|
|
GreaterThanEqualsTo |
|
|
EqualsTo |
|
|
NotEqualsTo |
|
|
Pipe |
|
|
Or |
|
|
Ampersand |
|
|
And |
|
|
If |
|
|
Do |
|
|
While |
|
|
Until |
|
|
PlusAssign |
|
|
MinusAssign |
|
|
MultiplyAssign |
|
|
DivideAssign |
|
|
LeftShiftAssign |
|
|
RightShiftAssign |
|
|
AndAssign |
|
|
OrAssign |
|
|
XOrAssign |
|
|
LeftShift |
|
|
RightShift |
|
|
XOr |
|
|
Modulo |
|
|
ModuloAssign |
|
|
Return |
|
|
Throw |
|
|
PowerOf |
|
|
In |
|
|
PowerOfAssign => true,
|
|
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Get the precedence number of the token.
|
|
pub fn precedence(&self) -> u8 {
|
|
use Token::*;
|
|
|
|
match self {
|
|
// Assignments are not considered expressions - set to zero
|
|
Equals | PlusAssign | MinusAssign | MultiplyAssign | DivideAssign | LeftShiftAssign
|
|
| RightShiftAssign | AndAssign | OrAssign | XOrAssign | ModuloAssign
|
|
| PowerOfAssign => 0,
|
|
|
|
Or | XOr | Pipe => 30,
|
|
|
|
And | Ampersand => 60,
|
|
|
|
EqualsTo | NotEqualsTo => 90,
|
|
|
|
In => 110,
|
|
|
|
LessThan | LessThanEqualsTo | GreaterThan | GreaterThanEqualsTo => 130,
|
|
|
|
Plus | Minus => 150,
|
|
|
|
Divide | Multiply | Modulo => 180,
|
|
|
|
PowerOf => 190,
|
|
|
|
LeftShift | RightShift => 210,
|
|
|
|
Period => 240,
|
|
|
|
_ => 0,
|
|
}
|
|
}
|
|
|
|
/// Does an expression bind to the right (instead of left)?
|
|
pub fn is_bind_right(&self) -> bool {
|
|
use Token::*;
|
|
|
|
match self {
|
|
// Assignments bind to the right
|
|
Equals | PlusAssign | MinusAssign | MultiplyAssign | DivideAssign | LeftShiftAssign
|
|
| RightShiftAssign | AndAssign | OrAssign | XOrAssign | ModuloAssign
|
|
| PowerOfAssign => true,
|
|
|
|
// Property access binds to the right
|
|
Period => true,
|
|
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Is this token an operator?
|
|
pub fn is_operator(&self) -> bool {
|
|
use Token::*;
|
|
|
|
match self {
|
|
LeftBrace | RightBrace | LeftParen | RightParen | LeftBracket | RightBracket | Plus
|
|
| UnaryPlus | Minus | UnaryMinus | Multiply | Divide | Modulo | PowerOf | LeftShift
|
|
| RightShift | SemiColon | Colon | DoubleColon | Comma | Period | MapStart | Equals
|
|
| LessThan | GreaterThan | LessThanEqualsTo | GreaterThanEqualsTo | EqualsTo
|
|
| NotEqualsTo | Bang | Pipe | Or | XOr | Ampersand | And | PlusAssign | MinusAssign
|
|
| MultiplyAssign | DivideAssign | LeftShiftAssign | RightShiftAssign | AndAssign
|
|
| OrAssign | XOrAssign | ModuloAssign | PowerOfAssign => true,
|
|
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Is this token an active standard keyword?
|
|
pub fn is_keyword(&self) -> bool {
|
|
use Token::*;
|
|
|
|
match self {
|
|
#[cfg(not(feature = "no_function"))]
|
|
Fn | Private => true,
|
|
|
|
#[cfg(not(feature = "no_module"))]
|
|
Import | Export | As => true,
|
|
|
|
True | False | Let | Const | If | Else | Do | While | Until | Loop | For | In
|
|
| Continue | Break | Return | Throw | Try | Catch => true,
|
|
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Is this token a reserved symbol?
|
|
#[inline(always)]
|
|
pub fn is_reserved(&self) -> bool {
|
|
match self {
|
|
Self::Reserved(_) => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Convert a token into a function name, if possible.
|
|
#[cfg(not(feature = "no_function"))]
|
|
pub(crate) fn into_function_name_for_override(self) -> Result<String, Self> {
|
|
match self {
|
|
Self::Reserved(s) if can_override_keyword(&s) => Ok(s),
|
|
Self::Custom(s) | Self::Identifier(s) if is_valid_identifier(s.chars()) => Ok(s),
|
|
_ => Err(self),
|
|
}
|
|
}
|
|
|
|
/// Is this token a custom keyword?
|
|
#[inline(always)]
|
|
pub fn is_custom(&self) -> bool {
|
|
match self {
|
|
Self::Custom(_) => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<Token> for String {
|
|
#[inline(always)]
|
|
fn from(token: Token) -> Self {
|
|
token.syntax().into()
|
|
}
|
|
}
|
|
|
|
/// _(INTERNALS)_ State of the tokenizer.
|
|
/// Exported under the `internals` feature only.
|
|
///
|
|
/// ## WARNING
|
|
///
|
|
/// This type is volatile and may change.
|
|
#[derive(Debug, Clone, Eq, PartialEq, Default)]
|
|
pub struct TokenizeState {
|
|
/// Maximum length of a string (0 = unlimited).
|
|
pub max_string_size: usize,
|
|
/// Can the next token be a unary operator?
|
|
pub non_unary: bool,
|
|
/// Is the tokenizer currently inside a block comment?
|
|
pub comment_level: usize,
|
|
/// Return [`None`] at the end of the stream instead of [`Some(Token::EOF)`][Token::EOF]?
|
|
pub end_with_none: bool,
|
|
/// Include comments?
|
|
pub include_comments: bool,
|
|
}
|
|
|
|
/// _(INTERNALS)_ Trait that encapsulates a peekable character input stream.
|
|
/// Exported under the `internals` feature only.
|
|
///
|
|
/// ## WARNING
|
|
///
|
|
/// This trait is volatile and may change.
|
|
pub trait InputStream {
|
|
fn unread(&mut self, ch: char);
|
|
/// Get the next character
|
|
fn get_next(&mut self) -> Option<char>;
|
|
/// Peek the next character
|
|
fn peek_next(&mut self) -> Option<char>;
|
|
}
|
|
|
|
/// _(INTERNALS)_ Parse a string literal wrapped by `enclosing_char`.
|
|
/// Exported under the `internals` feature only.
|
|
///
|
|
/// ## WARNING
|
|
///
|
|
/// This type is volatile and may change.
|
|
pub fn parse_string_literal(
|
|
stream: &mut impl InputStream,
|
|
state: &mut TokenizeState,
|
|
pos: &mut Position,
|
|
enclosing_char: char,
|
|
) -> Result<String, (LexError, Position)> {
|
|
let mut result: StaticVec<char> = Default::default();
|
|
let mut escape: StaticVec<char> = Default::default();
|
|
|
|
let start = *pos;
|
|
|
|
loop {
|
|
let next_char = stream.get_next().ok_or((LERR::UnterminatedString, start))?;
|
|
|
|
pos.advance();
|
|
|
|
if state.max_string_size > 0 && result.len() > state.max_string_size {
|
|
return Err((LexError::StringTooLong(state.max_string_size), *pos));
|
|
}
|
|
|
|
match next_char {
|
|
// \...
|
|
'\\' if escape.is_empty() => {
|
|
escape.push('\\');
|
|
}
|
|
// \\
|
|
'\\' if !escape.is_empty() => {
|
|
escape.clear();
|
|
result.push('\\');
|
|
}
|
|
// \t
|
|
't' if !escape.is_empty() => {
|
|
escape.clear();
|
|
result.push('\t');
|
|
}
|
|
// \n
|
|
'n' if !escape.is_empty() => {
|
|
escape.clear();
|
|
result.push('\n');
|
|
}
|
|
// \r
|
|
'r' if !escape.is_empty() => {
|
|
escape.clear();
|
|
result.push('\r');
|
|
}
|
|
// \x??, \u????, \U????????
|
|
ch @ 'x' | ch @ 'u' | ch @ 'U' if !escape.is_empty() => {
|
|
let mut seq = escape.clone();
|
|
escape.clear();
|
|
seq.push(ch);
|
|
|
|
let mut out_val: u32 = 0;
|
|
let len = match ch {
|
|
'x' => 2,
|
|
'u' => 4,
|
|
'U' => 8,
|
|
_ => unreachable!(),
|
|
};
|
|
|
|
for _ in 0..len {
|
|
let c = stream.get_next().ok_or_else(|| {
|
|
(
|
|
LERR::MalformedEscapeSequence(seq.iter().cloned().collect()),
|
|
*pos,
|
|
)
|
|
})?;
|
|
|
|
seq.push(c);
|
|
pos.advance();
|
|
|
|
out_val *= 16;
|
|
out_val += c.to_digit(16).ok_or_else(|| {
|
|
(
|
|
LERR::MalformedEscapeSequence(seq.iter().cloned().collect()),
|
|
*pos,
|
|
)
|
|
})?;
|
|
}
|
|
|
|
result.push(char::from_u32(out_val).ok_or_else(|| {
|
|
(
|
|
LERR::MalformedEscapeSequence(seq.into_iter().collect()),
|
|
*pos,
|
|
)
|
|
})?);
|
|
}
|
|
|
|
// \{enclosing_char} - escaped
|
|
ch if enclosing_char == ch && !escape.is_empty() => {
|
|
escape.clear();
|
|
result.push(ch)
|
|
}
|
|
|
|
// Close wrapper
|
|
ch if enclosing_char == ch && escape.is_empty() => break,
|
|
|
|
// Unknown escape sequence
|
|
ch if !escape.is_empty() => {
|
|
escape.push(ch);
|
|
|
|
return Err((
|
|
LERR::MalformedEscapeSequence(escape.into_iter().collect()),
|
|
*pos,
|
|
));
|
|
}
|
|
|
|
// Cannot have new-lines inside string literals
|
|
'\n' => {
|
|
pos.rewind();
|
|
return Err((LERR::UnterminatedString, start));
|
|
}
|
|
|
|
// All other characters
|
|
ch => {
|
|
escape.clear();
|
|
result.push(ch);
|
|
}
|
|
}
|
|
}
|
|
|
|
let s = result.iter().collect::<String>();
|
|
|
|
if state.max_string_size > 0 && s.len() > state.max_string_size {
|
|
return Err((LexError::StringTooLong(state.max_string_size), *pos));
|
|
}
|
|
|
|
Ok(s)
|
|
}
|
|
|
|
/// Consume the next character.
|
|
#[inline(always)]
|
|
fn eat_next(stream: &mut impl InputStream, pos: &mut Position) -> Option<char> {
|
|
pos.advance();
|
|
stream.get_next()
|
|
}
|
|
|
|
/// Scan for a block comment until the end.
|
|
fn scan_comment(
|
|
stream: &mut impl InputStream,
|
|
state: &mut TokenizeState,
|
|
pos: &mut Position,
|
|
comment: &mut String,
|
|
) {
|
|
while let Some(c) = stream.get_next() {
|
|
pos.advance();
|
|
|
|
if state.include_comments {
|
|
comment.push(c);
|
|
}
|
|
|
|
match c {
|
|
'/' => {
|
|
if let Some(c2) = stream.get_next() {
|
|
if state.include_comments {
|
|
comment.push(c2);
|
|
}
|
|
if c2 == '*' {
|
|
state.comment_level += 1;
|
|
}
|
|
}
|
|
pos.advance();
|
|
}
|
|
'*' => {
|
|
if let Some(c2) = stream.get_next() {
|
|
if state.include_comments {
|
|
comment.push(c2);
|
|
}
|
|
if c2 == '/' {
|
|
state.comment_level -= 1;
|
|
}
|
|
}
|
|
pos.advance();
|
|
}
|
|
'\n' => pos.new_line(),
|
|
_ => (),
|
|
}
|
|
|
|
if state.comment_level == 0 {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// _(INTERNALS)_ Get the next token from the `stream`.
|
|
/// Exported under the `internals` feature only.
|
|
///
|
|
/// ## WARNING
|
|
///
|
|
/// This type is volatile and may change.
|
|
#[inline]
|
|
pub fn get_next_token(
|
|
stream: &mut impl InputStream,
|
|
state: &mut TokenizeState,
|
|
pos: &mut Position,
|
|
) -> Option<(Token, Position)> {
|
|
let result = get_next_token_inner(stream, state, pos);
|
|
|
|
// Save the last token's state
|
|
if let Some((ref token, _)) = result {
|
|
state.non_unary = !token.is_next_unary();
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
/// Test if the given character is a hex character.
|
|
#[inline(always)]
|
|
fn is_hex_char(c: char) -> bool {
|
|
match c {
|
|
'a'..='f' => true,
|
|
'A'..='F' => true,
|
|
'0'..='9' => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Test if the given character is an octal character.
|
|
#[inline(always)]
|
|
fn is_octal_char(c: char) -> bool {
|
|
match c {
|
|
'0'..='7' => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Test if the given character is a binary character.
|
|
#[inline(always)]
|
|
fn is_binary_char(c: char) -> bool {
|
|
match c {
|
|
'0' | '1' => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Get the next token.
|
|
fn get_next_token_inner(
|
|
stream: &mut impl InputStream,
|
|
state: &mut TokenizeState,
|
|
pos: &mut Position,
|
|
) -> Option<(Token, Position)> {
|
|
// Still inside a comment?
|
|
if state.comment_level > 0 {
|
|
let start_pos = *pos;
|
|
let mut comment = String::new();
|
|
scan_comment(stream, state, pos, &mut comment);
|
|
|
|
if state.include_comments {
|
|
return Some((Token::Comment(comment), start_pos));
|
|
}
|
|
}
|
|
|
|
let mut negated = false;
|
|
|
|
while let Some(c) = stream.get_next() {
|
|
pos.advance();
|
|
|
|
let start_pos = *pos;
|
|
|
|
match (c, stream.peek_next().unwrap_or('\0')) {
|
|
// \n
|
|
('\n', _) => pos.new_line(),
|
|
|
|
// digit ...
|
|
('0'..='9', _) => {
|
|
let mut result: StaticVec<char> = Default::default();
|
|
let mut radix_base: Option<u32> = None;
|
|
result.push(c);
|
|
|
|
while let Some(next_char) = stream.peek_next() {
|
|
match next_char {
|
|
'0'..='9' | '_' => {
|
|
result.push(next_char);
|
|
eat_next(stream, pos);
|
|
}
|
|
#[cfg(not(feature = "no_float"))]
|
|
'.' => {
|
|
stream.get_next().unwrap();
|
|
|
|
// Check if followed by digits or something that cannot start a property name
|
|
match stream.peek_next().unwrap_or('\0') {
|
|
// digits after period - accept the period
|
|
'0'..='9' => {
|
|
result.push(next_char);
|
|
pos.advance()
|
|
}
|
|
// _ - cannot follow a decimal point
|
|
'_' => {
|
|
stream.unread(next_char);
|
|
break;
|
|
}
|
|
// .. - reserved symbol, not a floating-point number
|
|
'.' => {
|
|
stream.unread(next_char);
|
|
break;
|
|
}
|
|
// symbol after period - probably a float
|
|
ch @ _ if !is_id_first_alphabetic(ch) => {
|
|
result.push(next_char);
|
|
pos.advance();
|
|
result.push('0');
|
|
}
|
|
// Not a floating-point number
|
|
_ => {
|
|
stream.unread(next_char);
|
|
break;
|
|
}
|
|
}
|
|
|
|
while let Some(next_char_in_float) = stream.peek_next() {
|
|
match next_char_in_float {
|
|
'0'..='9' | '_' => {
|
|
result.push(next_char_in_float);
|
|
eat_next(stream, pos);
|
|
}
|
|
_ => break,
|
|
}
|
|
}
|
|
}
|
|
// 0x????, 0o????, 0b????
|
|
ch @ 'x' | ch @ 'X' | ch @ 'o' | ch @ 'O' | ch @ 'b' | ch @ 'B'
|
|
if c == '0' =>
|
|
{
|
|
result.push(next_char);
|
|
eat_next(stream, pos);
|
|
|
|
let valid = match ch {
|
|
'x' | 'X' => is_hex_char,
|
|
'o' | 'O' => is_octal_char,
|
|
'b' | 'B' => is_binary_char,
|
|
_ => unreachable!(),
|
|
};
|
|
|
|
radix_base = Some(match ch {
|
|
'x' | 'X' => 16,
|
|
'o' | 'O' => 8,
|
|
'b' | 'B' => 2,
|
|
_ => unreachable!(),
|
|
});
|
|
|
|
while let Some(next_char_in_escape_seq) = stream.peek_next() {
|
|
if !valid(next_char_in_escape_seq) {
|
|
break;
|
|
}
|
|
|
|
result.push(next_char_in_escape_seq);
|
|
eat_next(stream, pos);
|
|
}
|
|
}
|
|
|
|
_ => break,
|
|
}
|
|
}
|
|
|
|
if negated {
|
|
result.insert(0, '-');
|
|
}
|
|
|
|
// Parse number
|
|
if let Some(radix) = radix_base {
|
|
let out: String = result.iter().skip(2).filter(|&&c| c != '_').collect();
|
|
|
|
return Some((
|
|
INT::from_str_radix(&out, radix)
|
|
.map(Token::IntegerConstant)
|
|
.unwrap_or_else(|_| {
|
|
Token::LexError(LERR::MalformedNumber(result.into_iter().collect()))
|
|
}),
|
|
start_pos,
|
|
));
|
|
} else {
|
|
let out: String = result.iter().filter(|&&c| c != '_').collect();
|
|
let num = INT::from_str(&out).map(Token::IntegerConstant);
|
|
|
|
// If integer parsing is unnecessary, try float instead
|
|
#[cfg(not(feature = "no_float"))]
|
|
let num = num.or_else(|_| FLOAT::from_str(&out).map(Token::FloatConstant));
|
|
|
|
return Some((
|
|
num.unwrap_or_else(|_| {
|
|
Token::LexError(LERR::MalformedNumber(result.into_iter().collect()))
|
|
}),
|
|
start_pos,
|
|
));
|
|
}
|
|
}
|
|
|
|
// letter or underscore ...
|
|
('A'..='Z', _) | ('a'..='z', _) | ('_', _) => {
|
|
return get_identifier(stream, pos, start_pos, c);
|
|
}
|
|
|
|
// " - string literal
|
|
('"', _) => {
|
|
return parse_string_literal(stream, state, pos, '"').map_or_else(
|
|
|err| Some((Token::LexError(err.0), err.1)),
|
|
|out| Some((Token::StringConstant(out), start_pos)),
|
|
)
|
|
}
|
|
|
|
// ' - character literal
|
|
('\'', '\'') => {
|
|
return Some((
|
|
Token::LexError(LERR::MalformedChar("".to_string())),
|
|
start_pos,
|
|
))
|
|
}
|
|
('\'', _) => {
|
|
return Some(parse_string_literal(stream, state, pos, '\'').map_or_else(
|
|
|err| (Token::LexError(err.0), err.1),
|
|
|result| {
|
|
let mut chars = result.chars();
|
|
let first = chars.next().unwrap();
|
|
|
|
if chars.next().is_some() {
|
|
(Token::LexError(LERR::MalformedChar(result)), start_pos)
|
|
} else {
|
|
(Token::CharConstant(first), start_pos)
|
|
}
|
|
},
|
|
))
|
|
}
|
|
|
|
// Braces
|
|
('{', _) => return Some((Token::LeftBrace, start_pos)),
|
|
('}', _) => return Some((Token::RightBrace, start_pos)),
|
|
|
|
// Parentheses
|
|
('(', '*') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("(*".into()), start_pos));
|
|
}
|
|
('(', _) => return Some((Token::LeftParen, start_pos)),
|
|
(')', _) => return Some((Token::RightParen, start_pos)),
|
|
|
|
// Indexing
|
|
('[', _) => return Some((Token::LeftBracket, start_pos)),
|
|
(']', _) => return Some((Token::RightBracket, start_pos)),
|
|
|
|
// Map literal
|
|
#[cfg(not(feature = "no_object"))]
|
|
('#', '{') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::MapStart, start_pos));
|
|
}
|
|
('#', _) => return Some((Token::Reserved("#".into()), start_pos)),
|
|
|
|
// Operators
|
|
('+', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::PlusAssign, start_pos));
|
|
}
|
|
('+', '+') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("++".into()), start_pos));
|
|
}
|
|
('+', _) if !state.non_unary => return Some((Token::UnaryPlus, start_pos)),
|
|
('+', _) => return Some((Token::Plus, start_pos)),
|
|
|
|
('-', '0'..='9') if !state.non_unary => negated = true,
|
|
('-', '0'..='9') => return Some((Token::Minus, start_pos)),
|
|
('-', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::MinusAssign, start_pos));
|
|
}
|
|
('-', '>') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("->".into()), start_pos));
|
|
}
|
|
('-', '-') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("--".into()), start_pos));
|
|
}
|
|
('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)),
|
|
('-', _) => return Some((Token::Minus, start_pos)),
|
|
|
|
('*', '*') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("**".into()), start_pos));
|
|
}
|
|
('*', ')') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("*)".into()), start_pos));
|
|
}
|
|
('*', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::MultiplyAssign, start_pos));
|
|
}
|
|
('*', _) => return Some((Token::Multiply, start_pos)),
|
|
|
|
// Comments
|
|
('/', '/') => {
|
|
eat_next(stream, pos);
|
|
|
|
let mut comment = if state.include_comments {
|
|
"//".to_string()
|
|
} else {
|
|
String::new()
|
|
};
|
|
|
|
while let Some(c) = stream.get_next() {
|
|
if c == '\n' {
|
|
pos.new_line();
|
|
break;
|
|
}
|
|
|
|
if state.include_comments {
|
|
comment.push(c);
|
|
}
|
|
pos.advance();
|
|
}
|
|
|
|
if state.include_comments {
|
|
return Some((Token::Comment(comment), start_pos));
|
|
}
|
|
}
|
|
('/', '*') => {
|
|
state.comment_level = 1;
|
|
|
|
eat_next(stream, pos);
|
|
|
|
let mut comment = if state.include_comments {
|
|
"/*".to_string()
|
|
} else {
|
|
String::new()
|
|
};
|
|
scan_comment(stream, state, pos, &mut comment);
|
|
|
|
if state.include_comments {
|
|
return Some((Token::Comment(comment), start_pos));
|
|
}
|
|
}
|
|
|
|
('/', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::DivideAssign, start_pos));
|
|
}
|
|
('/', _) => return Some((Token::Divide, start_pos)),
|
|
|
|
(';', _) => return Some((Token::SemiColon, start_pos)),
|
|
(',', _) => return Some((Token::Comma, start_pos)),
|
|
|
|
('.', '.') => {
|
|
eat_next(stream, pos);
|
|
|
|
if stream.peek_next() == Some('.') {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("...".into()), start_pos));
|
|
} else {
|
|
return Some((Token::Reserved("..".into()), start_pos));
|
|
}
|
|
}
|
|
('.', _) => return Some((Token::Period, start_pos)),
|
|
|
|
('=', '=') => {
|
|
eat_next(stream, pos);
|
|
|
|
if stream.peek_next() == Some('=') {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("===".into()), start_pos));
|
|
}
|
|
|
|
return Some((Token::EqualsTo, start_pos));
|
|
}
|
|
('=', '>') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::DoubleArrow, start_pos));
|
|
}
|
|
('=', _) => return Some((Token::Equals, start_pos)),
|
|
|
|
(':', ':') => {
|
|
eat_next(stream, pos);
|
|
|
|
if stream.peek_next() == Some('<') {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("::<".into()), start_pos));
|
|
}
|
|
|
|
return Some((Token::DoubleColon, start_pos));
|
|
}
|
|
(':', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved(":=".into()), start_pos));
|
|
}
|
|
(':', _) => return Some((Token::Colon, start_pos)),
|
|
|
|
('<', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::LessThanEqualsTo, start_pos));
|
|
}
|
|
('<', '-') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("<-".into()), start_pos));
|
|
}
|
|
('<', '<') => {
|
|
eat_next(stream, pos);
|
|
|
|
return Some((
|
|
if stream.peek_next() == Some('=') {
|
|
eat_next(stream, pos);
|
|
Token::LeftShiftAssign
|
|
} else {
|
|
Token::LeftShift
|
|
},
|
|
start_pos,
|
|
));
|
|
}
|
|
('<', _) => return Some((Token::LessThan, start_pos)),
|
|
|
|
('>', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::GreaterThanEqualsTo, start_pos));
|
|
}
|
|
('>', '>') => {
|
|
eat_next(stream, pos);
|
|
|
|
return Some((
|
|
if stream.peek_next() == Some('=') {
|
|
eat_next(stream, pos);
|
|
Token::RightShiftAssign
|
|
} else {
|
|
Token::RightShift
|
|
},
|
|
start_pos,
|
|
));
|
|
}
|
|
('>', _) => return Some((Token::GreaterThan, start_pos)),
|
|
|
|
('!', '=') => {
|
|
eat_next(stream, pos);
|
|
|
|
if stream.peek_next() == Some('=') {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Reserved("!==".into()), start_pos));
|
|
}
|
|
|
|
return Some((Token::NotEqualsTo, start_pos));
|
|
}
|
|
('!', _) => return Some((Token::Bang, start_pos)),
|
|
|
|
('|', '|') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::Or, start_pos));
|
|
}
|
|
('|', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::OrAssign, start_pos));
|
|
}
|
|
('|', _) => return Some((Token::Pipe, start_pos)),
|
|
|
|
('&', '&') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::And, start_pos));
|
|
}
|
|
('&', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::AndAssign, start_pos));
|
|
}
|
|
('&', _) => return Some((Token::Ampersand, start_pos)),
|
|
|
|
('^', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::XOrAssign, start_pos));
|
|
}
|
|
('^', _) => return Some((Token::XOr, start_pos)),
|
|
|
|
('%', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::ModuloAssign, start_pos));
|
|
}
|
|
('%', _) => return Some((Token::Modulo, start_pos)),
|
|
|
|
('~', '=') => {
|
|
eat_next(stream, pos);
|
|
return Some((Token::PowerOfAssign, start_pos));
|
|
}
|
|
('~', _) => return Some((Token::PowerOf, start_pos)),
|
|
|
|
('@', _) => return Some((Token::Reserved("@".into()), start_pos)),
|
|
|
|
('$', _) => return Some((Token::Reserved("$".into()), start_pos)),
|
|
|
|
('\0', _) => unreachable!(),
|
|
|
|
(ch, _) if ch.is_whitespace() => (),
|
|
#[cfg(feature = "unicode-xid-ident")]
|
|
(ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => {
|
|
return get_identifier(stream, pos, start_pos, c);
|
|
}
|
|
(ch, _) => {
|
|
return Some((
|
|
Token::LexError(LERR::UnexpectedInput(ch.to_string())),
|
|
start_pos,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
pos.advance();
|
|
|
|
if state.end_with_none {
|
|
None
|
|
} else {
|
|
Some((Token::EOF, *pos))
|
|
}
|
|
}
|
|
|
|
/// Get the next identifier.
|
|
fn get_identifier(
|
|
stream: &mut impl InputStream,
|
|
pos: &mut Position,
|
|
start_pos: Position,
|
|
first_char: char,
|
|
) -> Option<(Token, Position)> {
|
|
let mut result: StaticVec<_> = Default::default();
|
|
result.push(first_char);
|
|
|
|
while let Some(next_char) = stream.peek_next() {
|
|
match next_char {
|
|
x if is_id_continue(x) => {
|
|
result.push(x);
|
|
eat_next(stream, pos);
|
|
}
|
|
_ => break,
|
|
}
|
|
}
|
|
|
|
let is_valid_identifier = is_valid_identifier(result.iter().cloned());
|
|
|
|
let identifier: String = result.into_iter().collect();
|
|
|
|
if let Some(token) = Token::lookup_from_syntax(&identifier) {
|
|
return Some((token, start_pos));
|
|
}
|
|
|
|
if !is_valid_identifier {
|
|
return Some((
|
|
Token::LexError(LERR::MalformedIdentifier(identifier)),
|
|
start_pos,
|
|
));
|
|
}
|
|
|
|
return Some((Token::Identifier(identifier), start_pos));
|
|
}
|
|
|
|
/// Is this keyword allowed as a function?
|
|
#[inline(always)]
|
|
pub fn is_keyword_function(name: &str) -> bool {
|
|
match name {
|
|
KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR
|
|
| KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Can this keyword be overridden as a function?
|
|
#[cfg(not(feature = "no_function"))]
|
|
#[inline(always)]
|
|
pub fn can_override_keyword(name: &str) -> bool {
|
|
match name {
|
|
KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
/// Is a text string a valid identifier?
|
|
pub fn is_valid_identifier(name: impl Iterator<Item = char>) -> bool {
|
|
let mut first_alphabetic = false;
|
|
|
|
for ch in name {
|
|
match ch {
|
|
'_' => (),
|
|
_ if is_id_first_alphabetic(ch) => first_alphabetic = true,
|
|
_ if !first_alphabetic => return false,
|
|
_ if char::is_ascii_alphanumeric(&ch) => (),
|
|
_ => return false,
|
|
}
|
|
}
|
|
|
|
first_alphabetic
|
|
}
|
|
|
|
#[cfg(feature = "unicode-xid-ident")]
|
|
#[inline(always)]
|
|
fn is_id_first_alphabetic(x: char) -> bool {
|
|
unicode_xid::UnicodeXID::is_xid_start(x)
|
|
}
|
|
|
|
#[cfg(feature = "unicode-xid-ident")]
|
|
#[inline(always)]
|
|
fn is_id_continue(x: char) -> bool {
|
|
unicode_xid::UnicodeXID::is_xid_continue(x)
|
|
}
|
|
|
|
#[cfg(not(feature = "unicode-xid-ident"))]
|
|
#[inline(always)]
|
|
fn is_id_first_alphabetic(x: char) -> bool {
|
|
x.is_ascii_alphabetic()
|
|
}
|
|
|
|
#[cfg(not(feature = "unicode-xid-ident"))]
|
|
#[inline(always)]
|
|
fn is_id_continue(x: char) -> bool {
|
|
x.is_ascii_alphanumeric() || x == '_'
|
|
}
|
|
|
|
/// A type that implements the [`InputStream`] trait.
|
|
/// Multiple character streams are jointed together to form one single stream.
|
|
pub struct MultiInputsStream<'a> {
|
|
/// Buffered character, if any.
|
|
buf: Option<char>,
|
|
/// The input character streams.
|
|
streams: StaticVec<Peekable<Chars<'a>>>,
|
|
/// The current stream index.
|
|
index: usize,
|
|
}
|
|
|
|
impl InputStream for MultiInputsStream<'_> {
|
|
/// Buffer a character.
|
|
#[inline(always)]
|
|
fn unread(&mut self, ch: char) {
|
|
self.buf = Some(ch);
|
|
}
|
|
/// Get the next character
|
|
fn get_next(&mut self) -> Option<char> {
|
|
if let Some(ch) = self.buf.take() {
|
|
return Some(ch);
|
|
}
|
|
|
|
loop {
|
|
if self.index >= self.streams.len() {
|
|
// No more streams
|
|
return None;
|
|
} else if let Some(ch) = self.streams[self.index].next() {
|
|
// Next character in current stream
|
|
return Some(ch);
|
|
} else {
|
|
// Jump to the next stream
|
|
self.index += 1;
|
|
}
|
|
}
|
|
}
|
|
/// Peek the next character
|
|
fn peek_next(&mut self) -> Option<char> {
|
|
if let Some(ch) = self.buf {
|
|
return Some(ch);
|
|
}
|
|
|
|
loop {
|
|
if self.index >= self.streams.len() {
|
|
// No more streams
|
|
return None;
|
|
} else if let Some(&ch) = self.streams[self.index].peek() {
|
|
// Next character in current stream
|
|
return Some(ch);
|
|
} else {
|
|
// Jump to the next stream
|
|
self.index += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An iterator on a [`Token`] stream.
|
|
pub struct TokenIterator<'a, 'e> {
|
|
/// Reference to the scripting `Engine`.
|
|
engine: &'e Engine,
|
|
/// Current state.
|
|
state: TokenizeState,
|
|
/// Current position.
|
|
pos: Position,
|
|
/// Input character stream.
|
|
stream: MultiInputsStream<'a>,
|
|
/// A processor function (if any) that maps a token to another.
|
|
map: Option<Box<dyn Fn(Token) -> Token>>,
|
|
}
|
|
|
|
impl<'a> Iterator for TokenIterator<'a, '_> {
|
|
type Item = (Token, Position);
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
let token = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) {
|
|
// {EOF}
|
|
None => None,
|
|
// Reserved keyword/symbol
|
|
Some((Token::Reserved(s), pos)) => Some((match
|
|
(s.as_str(), self.engine.custom_keywords.contains_key(&s))
|
|
{
|
|
("===", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(),
|
|
)),
|
|
("!==", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?".to_string(),
|
|
)),
|
|
("->", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'->' is not a valid symbol. This is not C or C++!".to_string())),
|
|
("<-", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(),
|
|
)),
|
|
(":=", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"':=' is not a valid assignment operator. This is not Go or Pascal! Should it be simply '='?".to_string(),
|
|
)),
|
|
("::<", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?".to_string(),
|
|
)),
|
|
("(*", false) | ("*)", false) | ("begin", false) | ("end", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?".to_string(),
|
|
)),
|
|
("#", false) => Token::LexError(LERR::ImproperSymbol(s,
|
|
"'#' is not a valid symbol. Should it be '#{'?".to_string(),
|
|
)),
|
|
// Reserved keyword/operator that is custom.
|
|
(_, true) => Token::Custom(s),
|
|
// Reserved operator that is not custom.
|
|
(token, false) if !is_valid_identifier(token.chars()) => {
|
|
let msg = format!("'{}' is a reserved symbol", token);
|
|
Token::LexError(LERR::ImproperSymbol(s, msg))
|
|
},
|
|
// Reserved keyword that is not custom and disabled.
|
|
(token, false) if self.engine.disabled_symbols.contains(token) => {
|
|
let msg = format!("reserved symbol '{}' is disabled", token);
|
|
Token::LexError(LERR::ImproperSymbol(s, msg))
|
|
},
|
|
// Reserved keyword/operator that is not custom.
|
|
(_, false) => Token::Reserved(s),
|
|
}, pos)),
|
|
// Custom keyword
|
|
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&s) => {
|
|
Some((Token::Custom(s), pos))
|
|
}
|
|
// Custom standard keyword - must be disabled
|
|
Some((token, pos)) if token.is_keyword() && self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => {
|
|
if self.engine.disabled_symbols.contains(token.syntax().as_ref()) {
|
|
// Disabled standard keyword
|
|
Some((Token::Custom(token.syntax().into()), pos))
|
|
} else {
|
|
// Active standard keyword - should never be a custom keyword!
|
|
unreachable!()
|
|
}
|
|
}
|
|
// Disabled operator
|
|
Some((token, pos)) if token.is_operator() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
|
|
Some((
|
|
Token::LexError(LexError::UnexpectedInput(token.syntax().into())),
|
|
pos,
|
|
))
|
|
}
|
|
// Disabled standard keyword
|
|
Some((token, pos)) if token.is_keyword() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
|
|
Some((Token::Reserved(token.syntax().into()), pos))
|
|
}
|
|
r => r,
|
|
};
|
|
|
|
match token {
|
|
None => None,
|
|
Some((token, pos)) => {
|
|
if let Some(ref map) = self.map {
|
|
Some((map(token), pos))
|
|
} else {
|
|
Some((token, pos))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Engine {
|
|
/// Tokenize an input text stream.
|
|
#[inline]
|
|
pub fn lex<'a, 'e>(
|
|
&'e self,
|
|
input: impl IntoIterator<Item = &'a &'a str>,
|
|
map: Option<Box<dyn Fn(Token) -> Token>>,
|
|
) -> TokenIterator<'a, 'e> {
|
|
TokenIterator {
|
|
engine: self,
|
|
state: TokenizeState {
|
|
#[cfg(not(feature = "unchecked"))]
|
|
max_string_size: self.limits.max_string_size,
|
|
#[cfg(feature = "unchecked")]
|
|
max_string_size: 0,
|
|
non_unary: false,
|
|
comment_level: 0,
|
|
end_with_none: false,
|
|
include_comments: false,
|
|
},
|
|
pos: Position::new(1, 0),
|
|
stream: MultiInputsStream {
|
|
buf: None,
|
|
streams: input.into_iter().map(|s| s.chars().peekable()).collect(),
|
|
index: 0,
|
|
},
|
|
map,
|
|
}
|
|
}
|
|
}
|