kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use Option<Token> instead of Token::NONE.

Browse Source
This commit is contained in:
Stephen Chung 2022-12-27 22:51:38 +08:00
parent bb1136e8ad
commit 00c434eb71
9 changed files with 77 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/api/custom_syntax.rs
View File

@ -4,7 +4,7 @@
use crate::ast::Expr;
use crate::func::SendSync;
use crate::parser::ParseResult;
use crate::tokenizer::{is_reserved_keyword_or_symbol, is_valid_identifier, Token, NO_TOKEN};
use crate::tokenizer::{is_reserved_keyword_or_symbol, is_valid_identifier, Token};
use crate::types::dynamic::Variant;
use crate::{
Dynamic, Engine, EvalContext, Identifier, ImmutableString, LexError, Position, RhaiResult,
@ -231,11 +231,11 @@ impl Engine {
continue;
}
let token = Token::lookup_symbol_from_syntax(s).unwrap_or_else(|| {
let token = Token::lookup_symbol_from_syntax(s).or_else(|| {
if is_reserved_keyword_or_symbol(s) {
Token::Reserved(Box::new(s.into()))
Some(Token::Reserved(Box::new(s.into())))
} else {
NO_TOKEN
None
}
});
@ -256,13 +256,13 @@ impl Engine {
#[cfg(not(feature = "no_float"))]
CUSTOM_SYNTAX_MARKER_FLOAT if !segments.is_empty() => s.into(),
// Standard or reserved keyword/symbol not in first position
_ if !segments.is_empty() && token != NO_TOKEN => {
_ if !segments.is_empty() && token.is_some() => {
// Make it a custom keyword/symbol if it is disabled or reserved
if (self
.disabled_symbols
.as_deref()
.map_or(false, |m| m.contains(s))
|| token.is_reserved())
|| token.as_ref().map_or(false, Token::is_reserved))
&& !self
.custom_keywords
.as_deref()
@ -276,7 +276,7 @@ impl Engine {
}
// Standard keyword in first position but not disabled
_ if segments.is_empty()
&& token.is_standard_keyword()
&& token.as_ref().map_or(false, Token::is_standard_keyword)
&& !self
.disabled_symbols
.as_deref()
@ -298,7 +298,7 @@ impl Engine {
.disabled_symbols
.as_deref()
.map_or(false, |m| m.contains(s))
|| (token.is_reserved()
|| (token.as_ref().map_or(false, Token::is_reserved)
&& !self
.custom_keywords
.as_deref()

9
src/ast/expr.rs
View File

@ -3,7 +3,7 @@
use super::{ASTFlags, ASTNode, Ident, Namespace, Stmt, StmtBlock};
use crate::engine::{KEYWORD_FN_PTR, OP_EXCLUSIVE_RANGE, OP_INCLUSIVE_RANGE};
use crate::func::hashing::ALT_ZERO_HASH;
use crate::tokenizer::{Token, NO_TOKEN};
use crate::tokenizer::Token;
use crate::types::dynamic::Union;
use crate::{
calc_fn_hash, Dynamic, FnPtr, Identifier, ImmutableString, Position, SmartString, StaticVec,
@ -207,8 +207,7 @@ pub struct FnCallExpr {
/// Does this function call capture the parent scope?
pub capture_parent_scope: bool,
/// Is this function call a native operator?
/// Otherwise set to [`Token::NONE`].
pub op_token: Token,
pub op_token: Option<Token>,
}
impl fmt::Debug for FnCallExpr {
@ -222,7 +221,7 @@ impl fmt::Debug for FnCallExpr {
ff.field("hash", &self.hashes)
.field("name", &self.name)
.field("args", &self.args);
if self.op_token != NO_TOKEN {
if self.op_token.is_some() {
ff.field("op_token", &self.op_token);
}
if self.capture_parent_scope {
@ -582,7 +581,7 @@ impl Expr {
hashes: calc_fn_hash(None, f.fn_name(), 1).into(),
args: once(Self::StringConstant(f.fn_name().into(), pos)).collect(),
capture_parent_scope: false,
op_token: NO_TOKEN,
op_token: None,
}
.into(),
pos,

18
src/eval/chaining.rs
View File

@ -5,7 +5,6 @@ use super::{Caches, GlobalRuntimeState, Target};
use crate::ast::{ASTFlags, BinaryExpr, Expr, OpAssignment};
use crate::config::hashing::SusLock;
use crate::engine::{FN_IDX_GET, FN_IDX_SET};
use crate::tokenizer::NO_TOKEN;
use crate::types::dynamic::Union;
use crate::{
calc_fn_hash, Dynamic, Engine, FnArgsVec, Position, RhaiResult, RhaiResultOf, Scope, ERR,
@ -73,7 +72,7 @@ impl Engine {
let hash = hash_idx().0;
let args = &mut [target, idx];
self.exec_native_fn_call(global, caches, FN_IDX_GET, NO_TOKEN, hash, args, true, pos)
self.exec_native_fn_call(global, caches, FN_IDX_GET, None, hash, args, true, pos)
.map(|(r, ..)| r)
}
@ -95,7 +94,7 @@ impl Engine {
let args = &mut [target, idx, new_val];
self.exec_native_fn_call(
global, caches, FN_IDX_SET, NO_TOKEN, hash, args, is_ref_mut, pos,
global, caches, FN_IDX_SET, None, hash, args, is_ref_mut, pos,
)
}
@ -751,8 +750,7 @@ impl Engine {
let (mut orig_val, ..) = self
.exec_native_fn_call(
global, caches, getter, NO_TOKEN, *hash_get, args, is_ref_mut,
*pos,
global, caches, getter, None, *hash_get, args, is_ref_mut, *pos,
)
.or_else(|err| match *err {
// Try an indexer if property does not exist
@ -786,7 +784,7 @@ impl Engine {
let args = &mut [target.as_mut(), &mut new_val];
self.exec_native_fn_call(
global, caches, setter, NO_TOKEN, *hash_set, args, is_ref_mut, *pos,
global, caches, setter, None, *hash_set, args, is_ref_mut, *pos,
)
.or_else(|err| match *err {
// Try an indexer if property does not exist
@ -813,7 +811,7 @@ impl Engine {
let args = &mut [target.as_mut()];
self.exec_native_fn_call(
global, caches, getter, NO_TOKEN, *hash_get, args, is_ref_mut, *pos,
global, caches, getter, None, *hash_get, args, is_ref_mut, *pos,
)
.map_or_else(
|err| match *err {
@ -904,8 +902,8 @@ impl Engine {
// Assume getters are always pure
let (mut val, ..) = self
.exec_native_fn_call(
global, caches, getter, NO_TOKEN, *hash_get, args,
is_ref_mut, pos,
global, caches, getter, None, *hash_get, args, is_ref_mut,
pos,
)
.or_else(|err| match *err {
// Try an indexer if property does not exist
@ -940,7 +938,7 @@ impl Engine {
// The return value is thrown away and not used.
let _ = self
.exec_native_fn_call(
global, caches, setter, NO_TOKEN, *hash_set, args,
global, caches, setter, None, *hash_set, args,
is_ref_mut, pos,
)
.or_else(|err| match *err {

4
src/eval/stmt.rs
View File

@ -159,7 +159,7 @@ impl Engine {
let op_assign = op_assign_token.literal_syntax();
let op = op_token.literal_syntax();
let token = op_assign_token.clone();
let token = Some(op_assign_token.clone());
match self
.exec_native_fn_call(global, caches, op_assign, token, hash, args, true, *op_pos)
@ -168,7 +168,7 @@ impl Engine {
Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) =>
{
// Expand to `var = var op rhs`
let token = op_token.clone();
let token = Some(op_token.clone());
*args[0] = self
.exec_native_fn_call(

47
src/func/call.rs
View File

@ -8,7 +8,7 @@ use crate::engine::{
KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_TYPE_OF,
};
use crate::eval::{Caches, FnResolutionCacheEntry, GlobalRuntimeState};
use crate::tokenizer::{is_valid_function_name, Token, NO_TOKEN};
use crate::tokenizer::{is_valid_function_name, Token};
use crate::{
calc_fn_hash, calc_fn_hash_full, Dynamic, Engine, FnArgsVec, FnPtr, ImmutableString,
OptimizationLevel, Position, RhaiError, RhaiResult, RhaiResultOf, Scope, Shared, ERR,
@ -164,7 +164,7 @@ impl Engine {
_global: &GlobalRuntimeState,
caches: &'s mut Caches,
local_entry: &'s mut Option<FnResolutionCacheEntry>,
op_token: Token,
op_token: Option<Token>,
hash_base: u64,
args: Option<&mut FnCallArgs>,
allow_dynamic: bool,
@ -270,30 +270,29 @@ impl Engine {
}
// Try to find a built-in version
let builtin = args.and_then(|args| match op_token {
Token::NONE => None,
token if token.is_op_assignment() => {
let builtin =
args.and_then(|args| match op_token {
None => None,
Some(token) if token.is_op_assignment() => {
let (first_arg, rest_args) = args.split_first().unwrap();
get_builtin_op_assignment_fn(token, first_arg, rest_args[0]).map(
|(f, has_context)| FnResolutionCacheEntry {
get_builtin_op_assignment_fn(token, first_arg, rest_args[0])
.map(|(f, has_context)| FnResolutionCacheEntry {
func: CallableFunction::Method {
func: Shared::new(f),
has_context,
},
source: None,
},
)
})
}
token => get_builtin_binary_op_fn(token, args[0], args[1]).map(
|(f, has_context)| FnResolutionCacheEntry {
Some(token) => get_builtin_binary_op_fn(token, args[0], args[1])
.map(|(f, has_context)| FnResolutionCacheEntry {
func: CallableFunction::Method {
func: Shared::new(f),
has_context,
},
source: None,
},
),
}),
});
return if cache.filter.is_absent_and_set(hash) {
@ -346,7 +345,7 @@ impl Engine {
global: &mut GlobalRuntimeState,
caches: &mut Caches,
name: &str,
op_token: Token,
op_token: Option<Token>,
hash: u64,
args: &mut FnCallArgs,
is_ref_mut: bool,
@ -568,7 +567,7 @@ impl Engine {
caches: &mut Caches,
_scope: Option<&mut Scope>,
fn_name: &str,
op_token: Token,
op_token: Option<Token>,
hashes: FnCallHashes,
mut _args: &mut FnCallArgs,
is_ref_mut: bool,
@ -634,7 +633,7 @@ impl Engine {
let local_entry = &mut None;
if let Some(FnResolutionCacheEntry { func, ref source }) = self
.resolve_fn(global, caches, local_entry, NO_TOKEN, hash, None, false)
.resolve_fn(global, caches, local_entry, None, hash, None, false)
.cloned()
{
// Script function call
@ -801,7 +800,7 @@ impl Engine {
caches,
None,
fn_name,
NO_TOKEN,
None,
new_hash,
args,
false,
@ -888,7 +887,7 @@ impl Engine {
caches,
None,
&fn_name,
NO_TOKEN,
None,
new_hash,
args,
is_ref_mut,
@ -975,7 +974,7 @@ impl Engine {
caches,
None,
fn_name,
NO_TOKEN,
None,
hash,
&mut args,
is_ref_mut,
@ -1001,7 +1000,7 @@ impl Engine {
scope: &mut Scope,
mut this_ptr: Option<&mut Dynamic>,
fn_name: &str,
op_token: Token,
op_token: Option<Token>,
first_arg: Option<&Expr>,
args_expr: &[Expr],
hashes: FnCallHashes,
@ -1017,7 +1016,7 @@ impl Engine {
let redirected; // Handle call() - Redirect function call
match name {
_ if op_token != NO_TOKEN => (),
_ if op_token.is_some() => (),
// Handle call(fn_ptr, ...)
KEYWORD_FN_PTR_CALL if total_args >= 1 => {
@ -1571,7 +1570,7 @@ impl Engine {
let op_token = op_token.clone();
// Short-circuit native unary operator call if under Fast Operators mode
if op_token == Token::Bang && self.fast_operators() && args.len() == 1 {
if op_token == Some(Token::Bang) && self.fast_operators() && args.len() == 1 {
let mut value = self
.get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])?
.0
@ -1587,7 +1586,7 @@ impl Engine {
}
// Short-circuit native binary operator call if under Fast Operators mode
if op_token != NO_TOKEN && self.fast_operators() && args.len() == 2 {
if op_token.is_some() && self.fast_operators() && args.len() == 2 {
let mut lhs = self
.get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])?
.0
@ -1601,7 +1600,7 @@ impl Engine {
let operands = &mut [&mut lhs, &mut rhs];
if let Some((func, need_context)) =
get_builtin_binary_op_fn(op_token.clone(), operands[0], operands[1])
get_builtin_binary_op_fn(op_token.clone().unwrap(), operands[0], operands[1])
{
// Built-in found
auto_restore! { let orig_level = global.level; global.level += 1 }

4
src/func/native.rs
View File

@ -4,7 +4,7 @@ use super::call::FnCallArgs;
use crate::ast::FnCallHashes;
use crate::eval::{Caches, GlobalRuntimeState};
use crate::plugin::PluginFunction;
use crate::tokenizer::{is_valid_function_name, Token, TokenizeState, NO_TOKEN};
use crate::tokenizer::{is_valid_function_name, Token, TokenizeState};
use crate::types::dynamic::Variant;
use crate::{
calc_fn_hash, Dynamic, Engine, EvalContext, FuncArgs, Position, RhaiResult, RhaiResultOf,
@ -436,7 +436,7 @@ impl<'a> NativeCallContext<'a> {
let caches = &mut Caches::new();
let fn_name = fn_name.as_ref();
let op_token = Token::lookup_symbol_from_syntax(fn_name).unwrap_or(NO_TOKEN);
let op_token = Token::lookup_symbol_from_syntax(fn_name);
let args_len = args.len();
if native_only {

6
src/optimizer.rs
View File

@ -147,7 +147,7 @@ impl<'a> OptimizerState<'a> {
pub fn call_fn_with_constant_arguments(
&mut self,
fn_name: &str,
op_token: Token,
op_token: Option<Token>,
arg_values: &mut [Dynamic],
) -> Option<Dynamic> {
self.engine
@ -1137,8 +1137,8 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) {
return;
}
// Overloaded operators can override built-in.
_ if x.args.len() == 2 && x.op_token != Token::NONE && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => {
if let Some(result) = get_builtin_binary_op_fn(x.op_token.clone(), &arg_values[0], &arg_values[1])
_ if x.args.len() == 2 && x.op_token.is_some() && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => {
if let Some(result) = get_builtin_binary_op_fn(x.op_token.clone().unwrap(), &arg_values[0], &arg_values[1])
.and_then(|(f, ctx)| {
let context = if ctx {
Some((state.engine, x.name.as_str(), None, &state.global, *pos).into())

37
src/parser.rs
View File

@ -12,7 +12,7 @@ use crate::eval::{Caches, GlobalRuntimeState};
use crate::func::{hashing::get_hasher, StraightHashMap};
use crate::tokenizer::{
is_keyword_function, is_valid_function_name, is_valid_identifier, Token, TokenStream,
TokenizerControl, NO_TOKEN,
TokenizerControl,
};
use crate::types::dynamic::AccessMode;
use crate::types::StringsInterner;
@ -627,7 +627,7 @@ impl Engine {
return Ok(FnCallExpr {
name: state.get_interned_string(id),
capture_parent_scope,
op_token: NO_TOKEN,
op_token: None,
namespace: _namespace,
hashes,
args,
@ -702,7 +702,7 @@ impl Engine {
return Ok(FnCallExpr {
name: state.get_interned_string(id),
capture_parent_scope,
op_token: NO_TOKEN,
op_token: None,
namespace: _namespace,
hashes,
args,
@ -1927,7 +1927,7 @@ impl Engine {
name: state.get_interned_string("-"),
hashes: FnCallHashes::from_native(calc_fn_hash(None, "-", 1)),
args,
op_token: token,
op_token: Some(token),
capture_parent_scope: false,
}
.into_fn_call_expr(pos))
@ -1955,7 +1955,7 @@ impl Engine {
name: state.get_interned_string("+"),
hashes: FnCallHashes::from_native(calc_fn_hash(None, "+", 1)),
args,
op_token: token,
op_token: Some(token),
capture_parent_scope: false,
}
.into_fn_call_expr(pos))
@ -1976,7 +1976,7 @@ impl Engine {
name: state.get_interned_string("!"),
hashes: FnCallHashes::from_native(calc_fn_hash(None, "!", 1)),
args,
op_token: token,
op_token: Some(token),
capture_parent_scope: false,
}
.into_fn_call_expr(pos))
@ -1990,7 +1990,7 @@ impl Engine {
/// Make an assignment statement.
fn make_assignment_stmt(
op: Token,
op: Option<Token>,
state: &mut ParseState,
lhs: Expr,
rhs: Expr,
@ -2023,10 +2023,10 @@ impl Engine {
}
}
let op_info = if op == NO_TOKEN {
OpAssignment::new_assignment(op_pos)
} else {
let op_info = if let Some(op) = op {
OpAssignment::new_op_assignment_from_token(op, op_pos)
} else {
OpAssignment::new_assignment(op_pos)
};
match lhs {
@ -2307,9 +2307,9 @@ impl Engine {
let hash = calc_fn_hash(None, &op, 2);
let is_valid_script_function = is_valid_function_name(&op);
let operator_token = if is_valid_script_function {
NO_TOKEN
None
} else {
op_token.clone()
Some(op_token.clone())
};
let mut args = StaticVec::new_const();
@ -2380,7 +2380,7 @@ impl Engine {
name: state.get_interned_string(op),
hashes: FnCallHashes::from_native(calc_fn_hash(None, op, 1)),
args,
op_token: Token::Bang,
op_token: Some(Token::Bang),
capture_parent_scope: false,
};
not_base.into_fn_call_expr(pos)
@ -3183,11 +3183,12 @@ impl Engine {
let (op, pos) = match input.peek().expect(NEVER_ENDS) {
// var = ...
(Token::Equals, ..) => (NO_TOKEN, eat_token(input, Token::Equals)),
(Token::Equals, ..) => (None, eat_token(input, Token::Equals)),
// var op= ...
(token, ..) if token.is_op_assignment() => {
input.next().map(|(op, pos)| (op, pos)).expect(NEVER_ENDS)
}
(token, ..) if token.is_op_assignment() => input
.next()
.map(|(op, pos)| (Some(op), pos))
.expect(NEVER_ENDS),
// Not op-assignment
_ => return Ok(Stmt::Expr(expr.into())),
};
@ -3683,7 +3684,7 @@ impl Engine {
num_externals + 1,
)),
args,
op_token: NO_TOKEN,
op_token: None,
capture_parent_scope: false,
}
.into_fn_call_expr(pos);

3
src/tokenizer.rs
View File

@ -55,9 +55,6 @@ type LERR = LexError;
/// Separator character for numbers.
const NUMBER_SEPARATOR: char = '_';
/// No token.
pub const NO_TOKEN: Token = Token::NONE;
/// A stream of tokens.
pub type TokenStream<'a> = Peekable<TokenIterator<'a>>;