kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Simplify ParseState.

Browse Source
This commit is contained in:
Stephen Chung 2022-02-28 16:32:08 +08:00
parent aee92f3b7b
commit d41d36c8bb
1 changed files with 68 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
src/parser.rs
View File

@ -34,10 +34,10 @@ pub type ParseResult<T> = Result<T, ParseError>;
type FnLib = BTreeMap<u64, Shared<ScriptFnDef>>; type FnLib = BTreeMap<u64, Shared<ScriptFnDef>>;
/// Invalid variable name that acts as a search barrier in a [`Scope`]. /// Invalid variable name that acts as a search barrier in a [`Scope`].
const SCOPE_SEARCH_BARRIER_MARKER: &str = "$BARRIER$"; const SCOPE_SEARCH_BARRIER_MARKER: &str = "$ BARRIER $";
/// The message: `TokenStream` never ends /// The message: `TokenStream` never ends
const NEVER_ENDS: &str = "`TokenStream` never ends"; const NEVER_ENDS: &str = "`Token`";
/// _(internals)_ A type that encapsulates the current state of the parser. /// _(internals)_ A type that encapsulates the current state of the parser.
/// Exported under the `internals` feature only. /// Exported under the `internals` feature only.
@ -46,7 +46,7 @@ pub struct ParseState<'e> {
/// Input stream buffer containing the next character to read. /// Input stream buffer containing the next character to read.
pub tokenizer_control: TokenizerControl, pub tokenizer_control: TokenizerControl,
/// Interned strings. /// Interned strings.
pub interned_strings: StringsInterner, interned_strings: StringsInterner,
/// Encapsulates a local stack with variable names to simulate an actual runtime scope. /// Encapsulates a local stack with variable names to simulate an actual runtime scope.
pub stack: Scope<'e>, pub stack: Scope<'e>,
/// Size of the local variables stack upon entry of the current block scope. /// Size of the local variables stack upon entry of the current block scope.
@ -63,27 +63,18 @@ pub struct ParseState<'e> {
/// Encapsulates a local stack with imported [module][crate::Module] names. /// Encapsulates a local stack with imported [module][crate::Module] names.
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
pub imports: StaticVec<Identifier>, pub imports: StaticVec<Identifier>,
/// Maximum levels of expression nesting. /// Maximum levels of expression nesting (0 for unlimited).
#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "unchecked"))]
pub max_expr_depth: Option<NonZeroUsize>, pub max_expr_depth: usize,
/// Maximum levels of expression nesting in functions.
#[cfg(not(feature = "unchecked"))]
#[cfg(not(feature = "no_function"))]
pub max_function_expr_depth: Option<NonZeroUsize>,
} }
impl<'e> ParseState<'e> { impl<'e> ParseState<'e> {
/// Create a new [`ParseState`]. /// Create a new [`ParseState`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub fn new(engine: &'e Engine, tokenizer_control: TokenizerControl) -> Self { pub fn new(engine: &Engine, tokenizer_control: TokenizerControl) -> Self {
Self { Self {
tokenizer_control, tokenizer_control,
#[cfg(not(feature = "unchecked"))]
max_expr_depth: NonZeroUsize::new(engine.max_expr_depth()),
#[cfg(not(feature = "unchecked"))]
#[cfg(not(feature = "no_function"))]
max_function_expr_depth: NonZeroUsize::new(engine.max_function_expr_depth()),
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
external_vars: Vec::new(), external_vars: Vec::new(),
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
@ -93,6 +84,8 @@ impl<'e> ParseState<'e> {
block_stack_len: 0, block_stack_len: 0,
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
imports: StaticVec::new_const(), imports: StaticVec::new_const(),
#[cfg(not(feature = "unchecked"))]
max_expr_depth: engine.max_expr_depth(),
} }
} }
@ -189,8 +182,6 @@ impl<'e> ParseState<'e> {
/// A type that encapsulates all the settings for a particular parsing function. /// A type that encapsulates all the settings for a particular parsing function.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
struct ParseSettings { struct ParseSettings {
/// Current position.
pos: Position,
/// Is the construct being parsed located at global level? /// Is the construct being parsed located at global level?
is_global: bool, is_global: bool,
/// Is the construct being parsed located at function definition level? /// Is the construct being parsed located at function definition level?
@ -199,24 +190,15 @@ struct ParseSettings {
/// Is the construct being parsed located inside a closure? /// Is the construct being parsed located inside a closure?
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
is_closure: bool, is_closure_scope: bool,
/// Is the current position inside a loop? /// Is the current position inside a loop?
is_breakable: bool, is_breakable: bool,
/// Default language options. /// Language options in effect (overrides Engine options).
default_options: LanguageOptions, options: LanguageOptions,
/// Is strict variables mode enabled?
strict_var: bool,
/// Is anonymous function allowed?
#[cfg(not(feature = "no_function"))]
allow_anonymous_fn: bool,
/// Is `if`-expression allowed?
allow_if_expr: bool,
/// Is `switch` expression allowed?
allow_switch_expr: bool,
/// Is statement-expression allowed?
allow_stmt_expr: bool,
/// Current expression nesting level. /// Current expression nesting level.
level: usize, level: usize,
/// Current position.
pos: Position,
} }
impl ParseSettings { impl ParseSettings {
@ -230,11 +212,13 @@ impl ParseSettings {
} }
} }
/// Make sure that the current level of expression nesting is within the maximum limit. /// Make sure that the current level of expression nesting is within the maximum limit.
///
/// If `limit` is zero, then checking is disabled.
#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "unchecked"))]
#[inline] #[inline]
pub fn ensure_level_within_max_limit(&self, limit: Option<NonZeroUsize>) -> ParseResult<()> { pub fn ensure_level_within_max_limit(&self, limit: usize) -> ParseResult<()> {
if let Some(limit) = limit { if limit > 0 {
if self.level > limit.get() { if self.level > limit {
return Err(PERR::ExprTooDeep.into_err(self.pos)); return Err(PERR::ExprTooDeep.into_err(self.pos));
} }
} }
@ -406,7 +390,7 @@ fn parse_symbol(input: &mut TokenStream) -> ParseResult<(SmartString, Position)>
(token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)), (token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)),
// Reserved symbol // Reserved symbol
(Token::Reserved(s), pos) if !is_valid_identifier(s.chars()) => Ok((s, pos)), (Token::Reserved(s), pos) if !is_valid_identifier(s.chars()) => Ok((s, pos)),
// Bad identifier // Bad symbol
(Token::LexError(err), pos) => Err(err.into_err(pos)), (Token::LexError(err), pos) => Err(err.into_err(pos)),
// Not a symbol // Not a symbol
(.., pos) => Err(PERR::MissingSymbol(String::new()).into_err(pos)), (.., pos) => Err(PERR::MissingSymbol(String::new()).into_err(pos)),
@ -436,11 +420,11 @@ impl Engine {
let expr = self.parse_expr(input, state, lib, settings.level_up())?; let expr = self.parse_expr(input, state, lib, settings.level_up())?;
match input.next().expect(NEVER_ENDS) { match input.next().expect(NEVER_ENDS) {
// ( xxx ) // ( ... )
(Token::RightParen, ..) => Ok(expr), (Token::RightParen, ..) => Ok(expr),
// ( <error> // ( <error>
(Token::LexError(err), pos) => Err(err.into_err(pos)), (Token::LexError(err), pos) => Err(err.into_err(pos)),
// ( xxx ??? // ( ... ???
(.., pos) => Err(PERR::MissingToken( (.., pos) => Err(PERR::MissingToken(
Token::RightParen.into(), Token::RightParen.into(),
"for a matching ( in this expression".into(), "for a matching ( in this expression".into(),
@ -493,7 +477,7 @@ impl Engine {
#[cfg(feature = "no_function")] #[cfg(feature = "no_function")]
let relax = false; let relax = false;
if !relax && settings.strict_var && index.is_none() { if !relax && settings.options.strict_var && index.is_none() {
return Err(PERR::ModuleUndefined(modules[0].name.to_string()) return Err(PERR::ModuleUndefined(modules[0].name.to_string())
.into_err(modules[0].pos)); .into_err(modules[0].pos));
} }
@ -554,7 +538,7 @@ impl Engine {
#[cfg(feature = "no_function")] #[cfg(feature = "no_function")]
let relax = false; let relax = false;
if !relax && settings.strict_var && index.is_none() { if !relax && settings.options.strict_var && index.is_none() {
return Err(PERR::ModuleUndefined(modules[0].name.to_string()) return Err(PERR::ModuleUndefined(modules[0].name.to_string())
.into_err(modules[0].pos)); .into_err(modules[0].pos));
} }
@ -1229,7 +1213,7 @@ impl Engine {
} }
// { - block statement as expression // { - block statement as expression
Token::LeftBrace if settings.allow_stmt_expr => { Token::LeftBrace if settings.options.allow_stmt_expr => {
match self.parse_block(input, state, lib, settings.level_up())? { match self.parse_block(input, state, lib, settings.level_up())? {
block @ Stmt::Block(..) => Expr::Stmt(Box::new(block.into())), block @ Stmt::Block(..) => Expr::Stmt(Box::new(block.into())),
stmt => unreachable!("Stmt::Block expected but gets {:?}", stmt), stmt => unreachable!("Stmt::Block expected but gets {:?}", stmt),
@ -1239,43 +1223,42 @@ impl Engine {
Token::LeftParen => self.parse_paren_expr(input, state, lib, settings.level_up())?, Token::LeftParen => self.parse_paren_expr(input, state, lib, settings.level_up())?,
// If statement is allowed to act as expressions // If statement is allowed to act as expressions
Token::If if settings.allow_if_expr => Expr::Stmt(Box::new( Token::If if settings.options.allow_if_expr => Expr::Stmt(Box::new(
self.parse_if(input, state, lib, settings.level_up())? self.parse_if(input, state, lib, settings.level_up())?
.into(), .into(),
)), )),
// Switch statement is allowed to act as expressions // Switch statement is allowed to act as expressions
Token::Switch if settings.allow_switch_expr => Expr::Stmt(Box::new( Token::Switch if settings.options.allow_switch_expr => Expr::Stmt(Box::new(
self.parse_switch(input, state, lib, settings.level_up())? self.parse_switch(input, state, lib, settings.level_up())?
.into(), .into(),
)), )),
// | ... // | ...
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
Token::Pipe | Token::Or if settings.allow_anonymous_fn => { Token::Pipe | Token::Or if settings.options.allow_anonymous_fn => {
let mut new_state = ParseState::new(self, state.tokenizer_control.clone()); let mut new_state = ParseState::new(self, state.tokenizer_control.clone());
#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "unchecked"))]
{ {
new_state.max_expr_depth = new_state.max_function_expr_depth; new_state.max_expr_depth = self.max_function_expr_depth();
} }
let new_settings = ParseSettings { let new_settings = ParseSettings {
allow_if_expr: settings.default_options.allow_if_expr,
allow_switch_expr: settings.default_options.allow_switch_expr,
allow_stmt_expr: settings.default_options.allow_stmt_expr,
allow_anonymous_fn: settings.default_options.allow_anonymous_fn,
strict_var: if cfg!(feature = "no_closure") {
settings.strict_var
} else {
// A capturing closure can access variables not defined locally
false
},
is_global: false, is_global: false,
is_function_scope: true, is_function_scope: true,
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
is_closure: true, is_closure_scope: true,
is_breakable: false, is_breakable: false,
level: 0, level: 0,
options: LanguageOptions {
strict_var: if cfg!(feature = "no_closure") {
settings.options.strict_var
} else {
// A capturing closure can access variables not defined locally
false
},
..self.options
},
..settings ..settings
}; };
@ -1286,7 +1269,10 @@ impl Engine {
|crate::ast::Ident { name, pos }| { |crate::ast::Ident { name, pos }| {
let index = state.access_var(name, *pos); let index = state.access_var(name, *pos);
if settings.strict_var && !settings.is_closure && index.is_none() { if settings.options.strict_var
&& !settings.is_closure_scope
&& index.is_none()
{
// If the parent scope is not inside another capturing closure // If the parent scope is not inside another capturing closure
// then we can conclude that the captured variable doesn't exist. // then we can conclude that the captured variable doesn't exist.
// Under Strict Variables mode, this is not allowed. // Under Strict Variables mode, this is not allowed.
@ -1429,7 +1415,7 @@ impl Engine {
_ => { _ => {
let index = state.access_var(&s, settings.pos); let index = state.access_var(&s, settings.pos);
if settings.strict_var && index.is_none() { if settings.options.strict_var && index.is_none() {
return Err( return Err(
PERR::VariableUndefined(s.to_string()).into_err(settings.pos) PERR::VariableUndefined(s.to_string()).into_err(settings.pos)
); );
@ -1663,7 +1649,7 @@ impl Engine {
#[cfg(feature = "no_function")] #[cfg(feature = "no_function")]
let relax = false; let relax = false;
if !relax && settings.strict_var && index.is_none() { if !relax && settings.options.strict_var && index.is_none() {
return Err(PERR::ModuleUndefined(namespace[0].name.to_string()) return Err(PERR::ModuleUndefined(namespace[0].name.to_string())
.into_err(namespace[0].pos)); .into_err(namespace[0].pos));
} }
@ -2685,8 +2671,7 @@ impl Engine {
// let name ... // let name ...
let (name, pos) = parse_var_name(input)?; let (name, pos) = parse_var_name(input)?;
if !settings.default_options.allow_shadowing && state.stack.iter().any(|(v, ..)| v == &name) if !self.allow_shadowing() && state.stack.iter().any(|(v, ..)| v == &name) {
{
return Err(PERR::VariableExists(name.to_string()).into_err(pos)); return Err(PERR::VariableExists(name.to_string()).into_err(pos));
} }
@ -3057,21 +3042,20 @@ impl Engine {
#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "unchecked"))]
{ {
new_state.max_expr_depth = new_state.max_function_expr_depth; new_state.max_expr_depth = self.max_function_expr_depth();
} }
let new_settings = ParseSettings { let new_settings = ParseSettings {
allow_if_expr: settings.default_options.allow_if_expr,
allow_switch_expr: settings.default_options.allow_switch_expr,
allow_stmt_expr: settings.default_options.allow_stmt_expr,
allow_anonymous_fn: settings.default_options.allow_anonymous_fn,
strict_var: settings.strict_var,
is_global: false, is_global: false,
is_function_scope: true, is_function_scope: true,
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
is_closure: false, is_closure_scope: false,
is_breakable: false, is_breakable: false,
level: 0, level: 0,
options: LanguageOptions {
strict_var: settings.options.strict_var,
..self.options
},
pos, pos,
..settings ..settings
}; };
@ -3112,25 +3096,25 @@ impl Engine {
Token::If => self.parse_if(input, state, lib, settings.level_up()), Token::If => self.parse_if(input, state, lib, settings.level_up()),
Token::Switch => self.parse_switch(input, state, lib, settings.level_up()), Token::Switch => self.parse_switch(input, state, lib, settings.level_up()),
Token::While | Token::Loop if settings.default_options.allow_looping => { Token::While | Token::Loop if self.allow_looping() => {
self.parse_while_loop(input, state, lib, settings.level_up()) self.parse_while_loop(input, state, lib, settings.level_up())
} }
Token::Do if settings.default_options.allow_looping => { Token::Do if self.allow_looping() => {
self.parse_do(input, state, lib, settings.level_up()) self.parse_do(input, state, lib, settings.level_up())
} }
Token::For if settings.default_options.allow_looping => { Token::For if self.allow_looping() => {
self.parse_for(input, state, lib, settings.level_up()) self.parse_for(input, state, lib, settings.level_up())
} }
Token::Continue if settings.default_options.allow_looping && settings.is_breakable => { Token::Continue if self.allow_looping() && settings.is_breakable => {
let pos = eat_token(input, Token::Continue); let pos = eat_token(input, Token::Continue);
Ok(Stmt::BreakLoop(ASTFlags::NONE, pos)) Ok(Stmt::BreakLoop(ASTFlags::NONE, pos))
} }
Token::Break if settings.default_options.allow_looping && settings.is_breakable => { Token::Break if self.allow_looping() && settings.is_breakable => {
let pos = eat_token(input, Token::Break); let pos = eat_token(input, Token::Break);
Ok(Stmt::BreakLoop(ASTFlags::BREAK, pos)) Ok(Stmt::BreakLoop(ASTFlags::BREAK, pos))
} }
Token::Continue | Token::Break if settings.default_options.allow_looping => { Token::Continue | Token::Break if self.allow_looping() => {
Err(PERR::LoopBreak.into_err(token_pos)) Err(PERR::LoopBreak.into_err(token_pos))
} }
@ -3540,21 +3524,22 @@ impl Engine {
let mut functions = BTreeMap::new(); let mut functions = BTreeMap::new();
let settings = ParseSettings { let settings = ParseSettings {
default_options: self.options,
allow_if_expr: false,
allow_switch_expr: false,
allow_stmt_expr: false,
#[cfg(not(feature = "no_function"))]
allow_anonymous_fn: false,
strict_var: self.options.strict_var,
is_global: true, is_global: true,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
is_function_scope: false, is_function_scope: false,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
is_closure: false, is_closure_scope: false,
is_breakable: false, is_breakable: false,
level: 0, level: 0,
options: LanguageOptions {
allow_if_expr: false,
allow_switch_expr: false,
allow_stmt_expr: false,
#[cfg(not(feature = "no_function"))]
allow_anonymous_fn: false,
..self.options
},
pos: Position::NONE, pos: Position::NONE,
}; };
let expr = self.parse_expr(input, state, &mut functions, settings)?; let expr = self.parse_expr(input, state, &mut functions, settings)?;
@ -3601,20 +3586,14 @@ impl Engine {
while !input.peek().expect(NEVER_ENDS).0.is_eof() { while !input.peek().expect(NEVER_ENDS).0.is_eof() {
let settings = ParseSettings { let settings = ParseSettings {
default_options: self.options,
allow_if_expr: self.options.allow_if_expr,
allow_switch_expr: self.options.allow_switch_expr,
allow_stmt_expr: self.options.allow_stmt_expr,
#[cfg(not(feature = "no_function"))]
allow_anonymous_fn: self.options.allow_anonymous_fn,
strict_var: self.options.strict_var,
is_global: true, is_global: true,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
is_function_scope: false, is_function_scope: false,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
is_closure: false, is_closure_scope: false,
is_breakable: false, is_breakable: false,
options: self.options,
level: 0, level: 0,
pos: Position::NONE, pos: Position::NONE,
}; };