kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Improve treatment of disabled symbols and custom symbols.

Browse Source
This commit is contained in:
Stephen Chung
2020-07-17 14:50:23 +08:00
parent d119e13b79
commit 3ae7cf4018
9 changed files with 110 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/parser.rs
View File

@@ -2151,9 +2151,6 @@ fn parse_expr(
exprs.push(Expr::Stmt(Box::new((stmt, pos))))
}
s => match input.peek().unwrap() {
(Token::Custom(custom), _) if custom == s => {
input.next().unwrap();
}
(t, _) if t.syntax().as_ref() == s => {
input.next().unwrap();
}

23
src/settings.rs
View File

@@ -2,7 +2,7 @@ use crate::engine::Engine;
use crate::module::ModuleResolver;
use crate::optimize::OptimizationLevel;
use crate::packages::PackageLibrary;
use crate::token::is_valid_identifier;
use crate::token::{is_valid_identifier, Token};
use crate::stdlib::{boxed::Box, format, string::String};
@@ -183,8 +183,7 @@ impl Engine {
/// engine.disable_symbol("if"); // disable the 'if' keyword
///
/// engine.compile("let x = if true { 42 } else { 0 };")?;
/// // ^ 'if' is parsed as a variable name
/// // ^ missing ';' after statement end
/// // ^ 'if' is rejected as a reserved keyword
/// # Ok(())
/// # }
/// ```
@@ -252,6 +251,24 @@ impl Engine {
return Err(format!("not a valid identifier: '{}'", keyword).into());
}
match Token::lookup_from_syntax(keyword) {
// Standard identifiers, reserved keywords and custom keywords are OK
None | Some(Token::Reserved(_)) | Some(Token::Custom(_)) => (),
// Disabled keywords are also OK
Some(token)
if !self
.disabled_symbols
.as_ref()
.map(|d| d.contains(token.syntax().as_ref()))
.unwrap_or(false) =>
{
()
}
// Active standard keywords cannot be made custom
Some(_) => return Err(format!("'{}' is a reserved keyword", keyword).into()),
}
// Add to custom keywords
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
}

5
src/syntax.rs
View File

@@ -88,11 +88,16 @@ impl Engine {
MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(),
// Standard symbols not in first position
s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
// Make it a custom keyword/operator if it is a disabled standard keyword/operator
// or a reserved keyword/operator.
if self
.disabled_symbols
.as_ref()
.map(|d| d.contains(s))
.unwrap_or(false)
|| Token::lookup_from_syntax(s)
.map(|token| token.is_reserved())
.unwrap_or(false)
{
// If symbol is disabled, make it a custom keyword
if self.custom_keywords.is_none() {

86
src/token.rs
View File

@@ -1334,73 +1334,81 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
self.engine.disabled_symbols.as_ref(),
self.engine.custom_keywords.as_ref(),
) {
// {EOF}
(None, _, _) => None,
(Some((Token::Reserved(s), pos)), None, None) => return Some((match s.as_str() {
"===" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?"
.to_string(),
// Reserved keyword/symbol
(Some((Token::Reserved(s), pos)), disabled, custom) => Some((match
(s.as_str(), custom.map(|c| c.contains_key(&s)).unwrap_or(false))
{
("===", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(),
))),
"!==" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?"
.to_string(),
("!==", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?".to_string(),
))),
"->" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'->' is not a valid symbol. This is not C or C++!".to_string(),
))),
"<-" => Token::LexError(Box::new(LERR::ImproperSymbol(
("->", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'->' is not a valid symbol. This is not C or C++!".to_string()))),
("<-", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(),
))),
"=>" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'=>' is not a valid symbol. This is not Rust! Should it be '>='?"
.to_string(),
("=>", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'=>' is not a valid symbol. This is not Rust! Should it be '>='?".to_string(),
))),
":=" => Token::LexError(Box::new(LERR::ImproperSymbol(
"':=' is not a valid assignment operator. This is not Go! Should it be simply '='?"
.to_string(),
(":=", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"':=' is not a valid assignment operator. This is not Go! Should it be simply '='?".to_string(),
))),
"::<" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?"
.to_string(),
("::<", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?".to_string(),
))),
"(*" | "*)" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?"
.to_string(),
("(*", false) | ("*)", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?".to_string(),
))),
"#" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'#' is not a valid symbol. Should it be '#{'?"
.to_string(),
("#", false) => Token::LexError(Box::new(LERR::ImproperSymbol(
"'#' is not a valid symbol. Should it be '#{'?".to_string(),
))),
token if !is_valid_identifier(token.chars()) => Token::LexError(Box::new(LERR::ImproperSymbol(
format!("'{}' is a reserved symbol.", token)
// Reserved keyword/operator that is custom.
(_, true) => Token::Custom(s),
// Reserved operator that is not custom.
(token, false) if !is_valid_identifier(token.chars()) => Token::LexError(Box::new(LERR::ImproperSymbol(
format!("'{}' is a reserved symbol", token)
))),
_ => Token::Reserved(s)
// Reserved keyword that is not custom and disabled.
(token, false) if disabled.map(|d| d.contains(token)).unwrap_or(false) => Token::LexError(Box::new(LERR::ImproperSymbol(
format!("reserved symbol '{}' is disabled", token)
))),
// Reserved keyword/operator that is not custom.
(_, false) => Token::Reserved(s),
}, pos)),
(r @ Some(_), None, None) => r,
// Custom keyword
(Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => {
// Convert custom keywords
Some((Token::Custom(s), pos))
}
(Some((token, pos)), _, Some(custom))
if (token.is_keyword() || token.is_operator() || token.is_reserved())
&& custom.contains_key(token.syntax().as_ref()) =>
// Custom standard keyword - must be disabled
(Some((token, pos)), Some(disabled), Some(custom))
if token.is_keyword() && custom.contains_key(token.syntax().as_ref()) =>
{
// Convert into custom keywords
Some((Token::Custom(token.syntax().into()), pos))
if disabled.contains(token.syntax().as_ref()) {
// Disabled standard keyword
Some((Token::Custom(token.syntax().into()), pos))
} else {
// Active standard keyword - should never be a custom keyword!
unreachable!()
}
}
// Disabled operator
(Some((token, pos)), Some(disabled), _)
if token.is_operator() && disabled.contains(token.syntax().as_ref()) =>
{
// Convert disallowed operators into lex errors
Some((
Token::LexError(Box::new(LexError::UnexpectedInput(token.syntax().into()))),
pos,
))
}
// Disabled standard keyword
(Some((token, pos)), Some(disabled), _)
if token.is_keyword() && disabled.contains(token.syntax().as_ref()) =>
{
// Convert disallowed keywords into identifiers
Some((Token::Identifier(token.syntax().into()), pos))
Some((Token::Reserved(token.syntax().into()), pos))
}
(r, _, _) => r,
}