diff --git a/src/api/custom_syntax.rs b/src/api/custom_syntax.rs index bc3e8d60..f881ff88 100644 --- a/src/api/custom_syntax.rs +++ b/src/api/custom_syntax.rs @@ -4,7 +4,7 @@ use crate::ast::Expr; use crate::func::SendSync; use crate::parser::ParseResult; -use crate::tokenizer::{is_reserved_keyword_or_symbol, is_valid_identifier, Token, NO_TOKEN}; +use crate::tokenizer::{is_reserved_keyword_or_symbol, is_valid_identifier, Token}; use crate::types::dynamic::Variant; use crate::{ Dynamic, Engine, EvalContext, Identifier, ImmutableString, LexError, Position, RhaiResult, @@ -231,11 +231,11 @@ impl Engine { continue; } - let token = Token::lookup_symbol_from_syntax(s).unwrap_or_else(|| { + let token = Token::lookup_symbol_from_syntax(s).or_else(|| { if is_reserved_keyword_or_symbol(s) { - Token::Reserved(Box::new(s.into())) + Some(Token::Reserved(Box::new(s.into()))) } else { - NO_TOKEN + None } }); @@ -256,13 +256,13 @@ impl Engine { #[cfg(not(feature = "no_float"))] CUSTOM_SYNTAX_MARKER_FLOAT if !segments.is_empty() => s.into(), // Standard or reserved keyword/symbol not in first position - _ if !segments.is_empty() && token != NO_TOKEN => { + _ if !segments.is_empty() && token.is_some() => { // Make it a custom keyword/symbol if it is disabled or reserved if (self .disabled_symbols .as_deref() .map_or(false, |m| m.contains(s)) - || token.is_reserved()) + || token.as_ref().map_or(false, Token::is_reserved)) && !self .custom_keywords .as_deref() @@ -276,7 +276,7 @@ impl Engine { } // Standard keyword in first position but not disabled _ if segments.is_empty() - && token.is_standard_keyword() + && token.as_ref().map_or(false, Token::is_standard_keyword) && !self .disabled_symbols .as_deref() @@ -298,7 +298,7 @@ impl Engine { .disabled_symbols .as_deref() .map_or(false, |m| m.contains(s)) - || (token.is_reserved() + || (token.as_ref().map_or(false, Token::is_reserved) && !self .custom_keywords .as_deref() diff --git a/src/ast/expr.rs b/src/ast/expr.rs index 36299d83..bfb4a9ef 100644 --- a/src/ast/expr.rs +++ b/src/ast/expr.rs @@ -3,7 +3,7 @@ use super::{ASTFlags, ASTNode, Ident, Namespace, Stmt, StmtBlock}; use crate::engine::{KEYWORD_FN_PTR, OP_EXCLUSIVE_RANGE, OP_INCLUSIVE_RANGE}; use crate::func::hashing::ALT_ZERO_HASH; -use crate::tokenizer::{Token, NO_TOKEN}; +use crate::tokenizer::Token; use crate::types::dynamic::Union; use crate::{ calc_fn_hash, Dynamic, FnPtr, Identifier, ImmutableString, Position, SmartString, StaticVec, @@ -207,8 +207,7 @@ pub struct FnCallExpr { /// Does this function call capture the parent scope? pub capture_parent_scope: bool, /// Is this function call a native operator? - /// Otherwise set to [`Token::NONE`]. - pub op_token: Token, + pub op_token: Option, } impl fmt::Debug for FnCallExpr { @@ -222,7 +221,7 @@ impl fmt::Debug for FnCallExpr { ff.field("hash", &self.hashes) .field("name", &self.name) .field("args", &self.args); - if self.op_token != NO_TOKEN { + if self.op_token.is_some() { ff.field("op_token", &self.op_token); } if self.capture_parent_scope { @@ -582,7 +581,7 @@ impl Expr { hashes: calc_fn_hash(None, f.fn_name(), 1).into(), args: once(Self::StringConstant(f.fn_name().into(), pos)).collect(), capture_parent_scope: false, - op_token: NO_TOKEN, + op_token: None, } .into(), pos, diff --git a/src/eval/chaining.rs b/src/eval/chaining.rs index 71d39a43..a31a4867 100644 --- a/src/eval/chaining.rs +++ b/src/eval/chaining.rs @@ -5,7 +5,6 @@ use super::{Caches, GlobalRuntimeState, Target}; use crate::ast::{ASTFlags, BinaryExpr, Expr, OpAssignment}; use crate::config::hashing::SusLock; use crate::engine::{FN_IDX_GET, FN_IDX_SET}; -use crate::tokenizer::NO_TOKEN; use crate::types::dynamic::Union; use crate::{ calc_fn_hash, Dynamic, Engine, FnArgsVec, Position, RhaiResult, RhaiResultOf, Scope, ERR, @@ -73,7 +72,7 @@ impl Engine { let hash = hash_idx().0; let args = &mut [target, idx]; - self.exec_native_fn_call(global, caches, FN_IDX_GET, NO_TOKEN, hash, args, true, pos) + self.exec_native_fn_call(global, caches, FN_IDX_GET, None, hash, args, true, pos) .map(|(r, ..)| r) } @@ -95,7 +94,7 @@ impl Engine { let args = &mut [target, idx, new_val]; self.exec_native_fn_call( - global, caches, FN_IDX_SET, NO_TOKEN, hash, args, is_ref_mut, pos, + global, caches, FN_IDX_SET, None, hash, args, is_ref_mut, pos, ) } @@ -751,8 +750,7 @@ impl Engine { let (mut orig_val, ..) = self .exec_native_fn_call( - global, caches, getter, NO_TOKEN, *hash_get, args, is_ref_mut, - *pos, + global, caches, getter, None, *hash_get, args, is_ref_mut, *pos, ) .or_else(|err| match *err { // Try an indexer if property does not exist @@ -786,7 +784,7 @@ impl Engine { let args = &mut [target.as_mut(), &mut new_val]; self.exec_native_fn_call( - global, caches, setter, NO_TOKEN, *hash_set, args, is_ref_mut, *pos, + global, caches, setter, None, *hash_set, args, is_ref_mut, *pos, ) .or_else(|err| match *err { // Try an indexer if property does not exist @@ -813,7 +811,7 @@ impl Engine { let args = &mut [target.as_mut()]; self.exec_native_fn_call( - global, caches, getter, NO_TOKEN, *hash_get, args, is_ref_mut, *pos, + global, caches, getter, None, *hash_get, args, is_ref_mut, *pos, ) .map_or_else( |err| match *err { @@ -904,8 +902,8 @@ impl Engine { // Assume getters are always pure let (mut val, ..) = self .exec_native_fn_call( - global, caches, getter, NO_TOKEN, *hash_get, args, - is_ref_mut, pos, + global, caches, getter, None, *hash_get, args, is_ref_mut, + pos, ) .or_else(|err| match *err { // Try an indexer if property does not exist @@ -940,7 +938,7 @@ impl Engine { // The return value is thrown away and not used. let _ = self .exec_native_fn_call( - global, caches, setter, NO_TOKEN, *hash_set, args, + global, caches, setter, None, *hash_set, args, is_ref_mut, pos, ) .or_else(|err| match *err { diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 49627c08..8bb47110 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -159,7 +159,7 @@ impl Engine { let op_assign = op_assign_token.literal_syntax(); let op = op_token.literal_syntax(); - let token = op_assign_token.clone(); + let token = Some(op_assign_token.clone()); match self .exec_native_fn_call(global, caches, op_assign, token, hash, args, true, *op_pos) @@ -168,7 +168,7 @@ impl Engine { Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) => { // Expand to `var = var op rhs` - let token = op_token.clone(); + let token = Some(op_token.clone()); *args[0] = self .exec_native_fn_call( diff --git a/src/func/call.rs b/src/func/call.rs index 61563777..b25568df 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -8,7 +8,7 @@ use crate::engine::{ KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_TYPE_OF, }; use crate::eval::{Caches, FnResolutionCacheEntry, GlobalRuntimeState}; -use crate::tokenizer::{is_valid_function_name, Token, NO_TOKEN}; +use crate::tokenizer::{is_valid_function_name, Token}; use crate::{ calc_fn_hash, calc_fn_hash_full, Dynamic, Engine, FnArgsVec, FnPtr, ImmutableString, OptimizationLevel, Position, RhaiError, RhaiResult, RhaiResultOf, Scope, Shared, ERR, @@ -164,7 +164,7 @@ impl Engine { _global: &GlobalRuntimeState, caches: &'s mut Caches, local_entry: &'s mut Option, - op_token: Token, + op_token: Option, hash_base: u64, args: Option<&mut FnCallArgs>, allow_dynamic: bool, @@ -270,31 +270,30 @@ impl Engine { } // Try to find a built-in version - let builtin = args.and_then(|args| match op_token { - Token::NONE => None, - token if token.is_op_assignment() => { - let (first_arg, rest_args) = args.split_first().unwrap(); + let builtin = + args.and_then(|args| match op_token { + None => None, + Some(token) if token.is_op_assignment() => { + let (first_arg, rest_args) = args.split_first().unwrap(); - get_builtin_op_assignment_fn(token, first_arg, rest_args[0]).map( - |(f, has_context)| FnResolutionCacheEntry { + get_builtin_op_assignment_fn(token, first_arg, rest_args[0]) + .map(|(f, has_context)| FnResolutionCacheEntry { + func: CallableFunction::Method { + func: Shared::new(f), + has_context, + }, + source: None, + }) + } + Some(token) => get_builtin_binary_op_fn(token, args[0], args[1]) + .map(|(f, has_context)| FnResolutionCacheEntry { func: CallableFunction::Method { func: Shared::new(f), has_context, }, source: None, - }, - ) - } - token => get_builtin_binary_op_fn(token, args[0], args[1]).map( - |(f, has_context)| FnResolutionCacheEntry { - func: CallableFunction::Method { - func: Shared::new(f), - has_context, - }, - source: None, - }, - ), - }); + }), + }); return if cache.filter.is_absent_and_set(hash) { // Do not cache "one-hit wonders" @@ -346,7 +345,7 @@ impl Engine { global: &mut GlobalRuntimeState, caches: &mut Caches, name: &str, - op_token: Token, + op_token: Option, hash: u64, args: &mut FnCallArgs, is_ref_mut: bool, @@ -568,7 +567,7 @@ impl Engine { caches: &mut Caches, _scope: Option<&mut Scope>, fn_name: &str, - op_token: Token, + op_token: Option, hashes: FnCallHashes, mut _args: &mut FnCallArgs, is_ref_mut: bool, @@ -634,7 +633,7 @@ impl Engine { let local_entry = &mut None; if let Some(FnResolutionCacheEntry { func, ref source }) = self - .resolve_fn(global, caches, local_entry, NO_TOKEN, hash, None, false) + .resolve_fn(global, caches, local_entry, None, hash, None, false) .cloned() { // Script function call @@ -801,7 +800,7 @@ impl Engine { caches, None, fn_name, - NO_TOKEN, + None, new_hash, args, false, @@ -888,7 +887,7 @@ impl Engine { caches, None, &fn_name, - NO_TOKEN, + None, new_hash, args, is_ref_mut, @@ -975,7 +974,7 @@ impl Engine { caches, None, fn_name, - NO_TOKEN, + None, hash, &mut args, is_ref_mut, @@ -1001,7 +1000,7 @@ impl Engine { scope: &mut Scope, mut this_ptr: Option<&mut Dynamic>, fn_name: &str, - op_token: Token, + op_token: Option, first_arg: Option<&Expr>, args_expr: &[Expr], hashes: FnCallHashes, @@ -1017,7 +1016,7 @@ impl Engine { let redirected; // Handle call() - Redirect function call match name { - _ if op_token != NO_TOKEN => (), + _ if op_token.is_some() => (), // Handle call(fn_ptr, ...) KEYWORD_FN_PTR_CALL if total_args >= 1 => { @@ -1571,7 +1570,7 @@ impl Engine { let op_token = op_token.clone(); // Short-circuit native unary operator call if under Fast Operators mode - if op_token == Token::Bang && self.fast_operators() && args.len() == 1 { + if op_token == Some(Token::Bang) && self.fast_operators() && args.len() == 1 { let mut value = self .get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])? .0 @@ -1587,7 +1586,7 @@ impl Engine { } // Short-circuit native binary operator call if under Fast Operators mode - if op_token != NO_TOKEN && self.fast_operators() && args.len() == 2 { + if op_token.is_some() && self.fast_operators() && args.len() == 2 { let mut lhs = self .get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])? .0 @@ -1601,7 +1600,7 @@ impl Engine { let operands = &mut [&mut lhs, &mut rhs]; if let Some((func, need_context)) = - get_builtin_binary_op_fn(op_token.clone(), operands[0], operands[1]) + get_builtin_binary_op_fn(op_token.clone().unwrap(), operands[0], operands[1]) { // Built-in found auto_restore! { let orig_level = global.level; global.level += 1 } diff --git a/src/func/native.rs b/src/func/native.rs index 7f0aba59..eaedd0ed 100644 --- a/src/func/native.rs +++ b/src/func/native.rs @@ -4,7 +4,7 @@ use super::call::FnCallArgs; use crate::ast::FnCallHashes; use crate::eval::{Caches, GlobalRuntimeState}; use crate::plugin::PluginFunction; -use crate::tokenizer::{is_valid_function_name, Token, TokenizeState, NO_TOKEN}; +use crate::tokenizer::{is_valid_function_name, Token, TokenizeState}; use crate::types::dynamic::Variant; use crate::{ calc_fn_hash, Dynamic, Engine, EvalContext, FuncArgs, Position, RhaiResult, RhaiResultOf, @@ -436,7 +436,7 @@ impl<'a> NativeCallContext<'a> { let caches = &mut Caches::new(); let fn_name = fn_name.as_ref(); - let op_token = Token::lookup_symbol_from_syntax(fn_name).unwrap_or(NO_TOKEN); + let op_token = Token::lookup_symbol_from_syntax(fn_name); let args_len = args.len(); if native_only { diff --git a/src/optimizer.rs b/src/optimizer.rs index 0b8ff895..15d11d2c 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -147,7 +147,7 @@ impl<'a> OptimizerState<'a> { pub fn call_fn_with_constant_arguments( &mut self, fn_name: &str, - op_token: Token, + op_token: Option, arg_values: &mut [Dynamic], ) -> Option { self.engine @@ -1137,8 +1137,8 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) { return; } // Overloaded operators can override built-in. - _ if x.args.len() == 2 && x.op_token != Token::NONE && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => { - if let Some(result) = get_builtin_binary_op_fn(x.op_token.clone(), &arg_values[0], &arg_values[1]) + _ if x.args.len() == 2 && x.op_token.is_some() && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => { + if let Some(result) = get_builtin_binary_op_fn(x.op_token.clone().unwrap(), &arg_values[0], &arg_values[1]) .and_then(|(f, ctx)| { let context = if ctx { Some((state.engine, x.name.as_str(), None, &state.global, *pos).into()) diff --git a/src/parser.rs b/src/parser.rs index 7201fb81..39c51d67 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -12,7 +12,7 @@ use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::{hashing::get_hasher, StraightHashMap}; use crate::tokenizer::{ is_keyword_function, is_valid_function_name, is_valid_identifier, Token, TokenStream, - TokenizerControl, NO_TOKEN, + TokenizerControl, }; use crate::types::dynamic::AccessMode; use crate::types::StringsInterner; @@ -627,7 +627,7 @@ impl Engine { return Ok(FnCallExpr { name: state.get_interned_string(id), capture_parent_scope, - op_token: NO_TOKEN, + op_token: None, namespace: _namespace, hashes, args, @@ -702,7 +702,7 @@ impl Engine { return Ok(FnCallExpr { name: state.get_interned_string(id), capture_parent_scope, - op_token: NO_TOKEN, + op_token: None, namespace: _namespace, hashes, args, @@ -1927,7 +1927,7 @@ impl Engine { name: state.get_interned_string("-"), hashes: FnCallHashes::from_native(calc_fn_hash(None, "-", 1)), args, - op_token: token, + op_token: Some(token), capture_parent_scope: false, } .into_fn_call_expr(pos)) @@ -1955,7 +1955,7 @@ impl Engine { name: state.get_interned_string("+"), hashes: FnCallHashes::from_native(calc_fn_hash(None, "+", 1)), args, - op_token: token, + op_token: Some(token), capture_parent_scope: false, } .into_fn_call_expr(pos)) @@ -1976,7 +1976,7 @@ impl Engine { name: state.get_interned_string("!"), hashes: FnCallHashes::from_native(calc_fn_hash(None, "!", 1)), args, - op_token: token, + op_token: Some(token), capture_parent_scope: false, } .into_fn_call_expr(pos)) @@ -1990,7 +1990,7 @@ impl Engine { /// Make an assignment statement. fn make_assignment_stmt( - op: Token, + op: Option, state: &mut ParseState, lhs: Expr, rhs: Expr, @@ -2023,10 +2023,10 @@ impl Engine { } } - let op_info = if op == NO_TOKEN { - OpAssignment::new_assignment(op_pos) - } else { + let op_info = if let Some(op) = op { OpAssignment::new_op_assignment_from_token(op, op_pos) + } else { + OpAssignment::new_assignment(op_pos) }; match lhs { @@ -2307,9 +2307,9 @@ impl Engine { let hash = calc_fn_hash(None, &op, 2); let is_valid_script_function = is_valid_function_name(&op); let operator_token = if is_valid_script_function { - NO_TOKEN + None } else { - op_token.clone() + Some(op_token.clone()) }; let mut args = StaticVec::new_const(); @@ -2380,7 +2380,7 @@ impl Engine { name: state.get_interned_string(op), hashes: FnCallHashes::from_native(calc_fn_hash(None, op, 1)), args, - op_token: Token::Bang, + op_token: Some(Token::Bang), capture_parent_scope: false, }; not_base.into_fn_call_expr(pos) @@ -3183,11 +3183,12 @@ impl Engine { let (op, pos) = match input.peek().expect(NEVER_ENDS) { // var = ... - (Token::Equals, ..) => (NO_TOKEN, eat_token(input, Token::Equals)), + (Token::Equals, ..) => (None, eat_token(input, Token::Equals)), // var op= ... - (token, ..) if token.is_op_assignment() => { - input.next().map(|(op, pos)| (op, pos)).expect(NEVER_ENDS) - } + (token, ..) if token.is_op_assignment() => input + .next() + .map(|(op, pos)| (Some(op), pos)) + .expect(NEVER_ENDS), // Not op-assignment _ => return Ok(Stmt::Expr(expr.into())), }; @@ -3683,7 +3684,7 @@ impl Engine { num_externals + 1, )), args, - op_token: NO_TOKEN, + op_token: None, capture_parent_scope: false, } .into_fn_call_expr(pos); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index f9d0c87f..1f309386 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -55,9 +55,6 @@ type LERR = LexError; /// Separator character for numbers. const NUMBER_SEPARATOR: char = '_'; -/// No token. -pub const NO_TOKEN: Token = Token::NONE; - /// A stream of tokens. pub type TokenStream<'a> = Peekable>;