From dbac4d56896426620fa54a25e61d472781f9beea Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 11 Feb 2023 23:51:36 +0800 Subject: [PATCH 01/41] Use StraightHashMap. --- src/ast/stmt.rs | 21 +++++++++++++++++---- src/parser.rs | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/ast/stmt.rs b/src/ast/stmt.rs index 139475cf..7110fd1b 100644 --- a/src/ast/stmt.rs +++ b/src/ast/stmt.rs @@ -2,6 +2,7 @@ use super::{ASTFlags, ASTNode, BinaryExpr, Expr, FnCallExpr, Ident}; use crate::engine::KEYWORD_EVAL; +use crate::func::StraightHashMap; use crate::tokenizer::Token; use crate::types::Span; use crate::{calc_fn_hash, Position, StaticVec, INT}; @@ -9,9 +10,8 @@ use crate::{calc_fn_hash, Position, StaticVec, INT}; use std::prelude::v1::*; use std::{ borrow::Borrow, - collections::BTreeMap, fmt, - hash::Hash, + hash::{Hash, Hasher}, mem, num::NonZeroUsize, ops::{Deref, DerefMut, Range, RangeInclusive}, @@ -303,18 +303,31 @@ pub type CaseBlocksList = smallvec::SmallVec<[usize; 1]>; /// _(internals)_ A type containing all cases for a `switch` statement. /// Exported under the `internals` feature only. -#[derive(Debug, Clone, Hash)] +#[derive(Debug, Clone)] pub struct SwitchCasesCollection { /// List of [`ConditionalExpr`]'s. pub expressions: StaticVec, /// Dictionary mapping value hashes to [`ConditionalExpr`]'s. - pub cases: BTreeMap, + pub cases: StraightHashMap, /// List of range cases. pub ranges: StaticVec, /// Statements block for the default case (there can be no condition for the default case). pub def_case: Option, } +impl Hash for SwitchCasesCollection { + #[inline(always)] + fn hash(&self, state: &mut H) { + self.expressions.hash(state); + + self.cases.len().hash(state); + self.cases.iter().for_each(|kv| kv.hash(state)); + + self.ranges.hash(state); + self.def_case.hash(state); + } +} + /// Number of items to keep inline for [`StmtBlockContainer`]. #[cfg(not(feature = "no_std"))] const STMT_BLOCK_INLINE_SIZE: usize = 8; diff --git a/src/parser.rs b/src/parser.rs index 9f8c6e1f..0f452d8c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1121,7 +1121,7 @@ impl Engine { } let mut expressions = StaticVec::::new(); - let mut cases = BTreeMap::::new(); + let mut cases = StraightHashMap::::default(); let mut ranges = StaticVec::::new(); let mut def_case = None; let mut def_case_pos = Position::NONE; From 8685323b1556dadf0dac5fc7a3d19aa80124e25b Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 11 Feb 2023 23:55:07 +0800 Subject: [PATCH 02/41] Use SmartString for comments. --- src/ast/script_fn.rs | 2 +- src/func/hashing.rs | 2 +- src/module/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ast/script_fn.rs b/src/ast/script_fn.rs index e42e1b8c..7126fdda 100644 --- a/src/ast/script_fn.rs +++ b/src/ast/script_fn.rs @@ -34,7 +34,7 @@ pub struct ScriptFnDef { /// /// Each line in non-block doc-comments starts with `///`. #[cfg(feature = "metadata")] - pub comments: Box<[crate::Identifier]>, + pub comments: Box<[crate::SmartString]>, } impl fmt::Display for ScriptFnDef { diff --git a/src/func/hashing.rs b/src/func/hashing.rs index b3e04613..ac0f1b27 100644 --- a/src/func/hashing.rs +++ b/src/func/hashing.rs @@ -27,7 +27,7 @@ impl Hasher for StraightHasher { fn finish(&self) -> u64 { self.0 } - #[inline(always)] + #[cold] fn write(&mut self, _bytes: &[u8]) { panic!("StraightHasher can only hash u64 values"); } diff --git a/src/module/mod.rs b/src/module/mod.rs index 962938ec..87b4f831 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -85,7 +85,7 @@ pub struct FuncInfoMetadata { pub return_type: Identifier, /// Comments. #[cfg(feature = "metadata")] - pub comments: Box<[Identifier]>, + pub comments: Box<[SmartString]>, } /// A type containing a single registered function. From 9f18d6519d598f75fe6b48e54be39c626b26479a Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 12 Feb 2023 00:13:54 +0800 Subject: [PATCH 03/41] Use debug_assert!. --- src/ast/expr.rs | 2 +- src/eval/expr.rs | 2 +- src/eval/stmt.rs | 2 +- src/func/call.rs | 4 ++-- src/tokenizer.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ast/expr.rs b/src/ast/expr.rs index 957a195e..586d5815 100644 --- a/src/ast/expr.rs +++ b/src/ast/expr.rs @@ -185,7 +185,7 @@ impl FnCallHashes { #[inline(always)] #[must_use] pub fn script(&self) -> u64 { - assert!(self.script.is_some()); + debug_assert!(self.script.is_some()); self.script.unwrap() } } diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 925a8132..0f3125f4 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -20,7 +20,7 @@ impl Engine { global: &GlobalRuntimeState, namespace: &crate::ast::Namespace, ) -> Option { - assert!(!namespace.is_empty()); + debug_assert!(!namespace.is_empty()); let root = namespace.root(); diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 417049b8..2fac1fe2 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -343,7 +343,7 @@ impl Engine { // First check hashes if let Some(case_blocks_list) = cases.get(&hash) { - assert!(!case_blocks_list.is_empty()); + debug_assert!(!case_blocks_list.is_empty()); for &index in case_blocks_list { let block = &expressions[index]; diff --git a/src/func/call.rs b/src/func/call.rs index bdcdf899..2eb7ace7 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -363,7 +363,7 @@ impl Engine { ); if let Some(FnResolutionCacheEntry { func, source }) = func { - assert!(func.is_native()); + debug_assert!(func.is_native()); // Push a new call stack frame #[cfg(feature = "debugging")] @@ -633,7 +633,7 @@ impl Engine { .cloned() { // Script function call - assert!(func.is_script()); + debug_assert!(func.is_script()); let f = func.get_script_fn_def().expect("script-defined function"); let environ = func.get_encapsulated_environ(); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 717b7282..4ec35bea 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -932,7 +932,7 @@ pub fn parse_string_literal( } loop { - assert!( + debug_assert!( !verbatim || escape.is_empty(), "verbatim strings should not have any escapes" ); From d0a47d7f664ac9f49fac15d4c0699299de85bd2c Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 12 Feb 2023 21:50:32 +0800 Subject: [PATCH 04/41] Refactor to do more pre-calculation work. --- src/api/register.rs | 2 +- src/ast/expr.rs | 2 +- src/ast/namespace.rs | 5 ++--- src/ast/stmt.rs | 36 ++++++++++++++++++++++++++++++------ src/engine.rs | 7 +++++++ src/eval/expr.rs | 4 ++-- src/eval/stmt.rs | 20 ++++++++------------ src/func/call.rs | 2 +- src/optimizer.rs | 7 +++---- src/packages/lang_core.rs | 5 +++-- src/parser.rs | 11 +++++++---- 11 files changed, 65 insertions(+), 36 deletions(-) diff --git a/src/api/register.rs b/src/api/register.rs index 203740a0..a2d26944 100644 --- a/src/api/register.rs +++ b/src/api/register.rs @@ -729,7 +729,7 @@ impl Engine { name: &str, module: SharedModule, ) { - let separator = crate::tokenizer::Token::DoubleColon.literal_syntax(); + let separator = crate::engine::NAMESPACE_SEPARATOR; if name.contains(separator) { let mut iter = name.splitn(2, separator); diff --git a/src/ast/expr.rs b/src/ast/expr.rs index 586d5815..0152b72a 100644 --- a/src/ast/expr.rs +++ b/src/ast/expr.rs @@ -383,7 +383,7 @@ impl fmt::Debug for Expr { #[cfg(not(feature = "no_module"))] if !x.1.is_empty() { - write!(f, "{}{}", x.1, Token::DoubleColon.literal_syntax())?; + write!(f, "{}{}", x.1, crate::engine::NAMESPACE_SEPARATOR)?; let pos = x.1.position(); if !pos.is_none() { display_pos = pos; diff --git a/src/ast/namespace.rs b/src/ast/namespace.rs index 2888b25e..c6cd91e9 100644 --- a/src/ast/namespace.rs +++ b/src/ast/namespace.rs @@ -2,7 +2,6 @@ #![cfg(not(feature = "no_module"))] use crate::ast::Ident; -use crate::tokenizer::Token; use crate::{Position, StaticVec}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -46,7 +45,7 @@ impl fmt::Debug for Namespace { .iter() .map(Ident::as_str) .collect::>() - .join(Token::DoubleColon.literal_syntax()), + .join(crate::engine::NAMESPACE_SEPARATOR), ) } } @@ -63,7 +62,7 @@ impl fmt::Display for Namespace { .iter() .map(Ident::as_str) .collect::>() - .join(Token::DoubleColon.literal_syntax()), + .join(crate::engine::NAMESPACE_SEPARATOR), ) } } diff --git a/src/ast/stmt.rs b/src/ast/stmt.rs index 7110fd1b..cd26df7f 100644 --- a/src/ast/stmt.rs +++ b/src/ast/stmt.rs @@ -1,7 +1,7 @@ //! Module defining script statements. use super::{ASTFlags, ASTNode, BinaryExpr, Expr, FnCallExpr, Ident}; -use crate::engine::KEYWORD_EVAL; +use crate::engine::{KEYWORD_EVAL, OP_EQUALS}; use crate::func::StraightHashMap; use crate::tokenizer::Token; use crate::types::Span; @@ -29,8 +29,12 @@ pub struct OpAssignment { hash_op: u64, /// Op-assignment operator. op_assign: Token, + /// Syntax of op-assignment operator. + op_assign_syntax: &'static str, /// Underlying operator. op: Token, + /// Syntax of underlying operator. + op_syntax: &'static str, /// [Position] of the op-assignment operator. pos: Position, } @@ -44,7 +48,9 @@ impl OpAssignment { hash_op_assign: 0, hash_op: 0, op_assign: Token::Equals, + op_assign_syntax: OP_EQUALS, op: Token::Equals, + op_syntax: OP_EQUALS, pos, } } @@ -56,17 +62,28 @@ impl OpAssignment { } /// Get information if this [`OpAssignment`] is an op-assignment. /// - /// Returns `( hash_op_assign, hash_op, op_assign, op )`: + /// Returns `( hash_op_assign, hash_op, op_assign, op_assign_syntax, op, op_syntax )`: /// /// * `hash_op_assign`: Hash of the op-assignment call. /// * `hash_op`: Hash of the underlying operator call (for fallback). /// * `op_assign`: Op-assignment operator. + /// * `op_assign_syntax`: Syntax of op-assignment operator. /// * `op`: Underlying operator. + /// * `op_syntax`: Syntax of underlying operator. #[must_use] #[inline] - pub fn get_op_assignment_info(&self) -> Option<(u64, u64, &Token, &Token)> { + pub fn get_op_assignment_info( + &self, + ) -> Option<(u64, u64, &Token, &'static str, &Token, &'static str)> { if self.is_op_assignment() { - Some((self.hash_op_assign, self.hash_op, &self.op_assign, &self.op)) + Some(( + self.hash_op_assign, + self.hash_op, + &self.op_assign, + self.op_assign_syntax, + &self.op, + self.op_syntax, + )) } else { None } @@ -99,11 +116,16 @@ impl OpAssignment { .get_base_op_from_assignment() .expect("op-assignment operator"); + let op_assign_syntax = op_assign.literal_syntax(); + let op_syntax = op.literal_syntax(); + Self { - hash_op_assign: calc_fn_hash(None, op_assign.literal_syntax(), 2), - hash_op: calc_fn_hash(None, op.literal_syntax(), 2), + hash_op_assign: calc_fn_hash(None, op_assign_syntax, 2), + hash_op: calc_fn_hash(None, op_syntax, 2), op_assign, + op_assign_syntax, op, + op_syntax, pos, } } @@ -139,7 +161,9 @@ impl fmt::Debug for OpAssignment { .field("hash_op_assign", &self.hash_op_assign) .field("hash_op", &self.hash_op) .field("op_assign", &self.op_assign) + .field("op_assign_syntax", &self.op_assign_syntax) .field("op", &self.op) + .field("op_syntax", &self.op_syntax) .field("pos", &self.pos) .finish() } else { diff --git a/src/engine.rs b/src/engine.rs index 3e07bda9..750b35f0 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -57,12 +57,19 @@ pub const OP_EQUALS: &str = Token::EqualsTo.literal_syntax(); /// The `in` operator is implemented as a call to this function. pub const OP_CONTAINS: &str = "contains"; +/// Standard not operator. +pub const OP_NOT: &str = Token::Bang.literal_syntax(); + /// Standard exclusive range operator. pub const OP_EXCLUSIVE_RANGE: &str = Token::ExclusiveRange.literal_syntax(); /// Standard inclusive range operator. pub const OP_INCLUSIVE_RANGE: &str = Token::InclusiveRange.literal_syntax(); +/// Separator for namespaces. +#[cfg(not(feature = "no_module"))] +pub const NAMESPACE_SEPARATOR: &str = Token::DoubleColon.literal_syntax(); + /// Rhai main scripting engine. /// /// # Thread Safety diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 0f3125f4..0567e39e 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -74,7 +74,7 @@ impl Engine { if let Some(module) = self.search_imports(global, ns) { return module.get_qualified_var(*hash_var).map_or_else( || { - let sep = crate::tokenizer::Token::DoubleColon.literal_syntax(); + let sep = crate::engine::NAMESPACE_SEPARATOR; Err(ERR::ErrorVariableNotFound( format!("{ns}{sep}{var_name}"), @@ -104,7 +104,7 @@ impl Engine { } } - let sep = crate::tokenizer::Token::DoubleColon.literal_syntax(); + let sep = crate::engine::NAMESPACE_SEPARATOR; return Err(ERR::ErrorVariableNotFound( format!("{ns}{sep}{var_name}"), diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 2fac1fe2..d6e5229e 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -127,21 +127,20 @@ impl Engine { let pos = op_info.position(); - if let Some((hash1, hash2, op_assign, op)) = op_info.get_op_assignment_info() { + if let Some((hash_x, hash, op_x, op_x_str, op, op_str)) = op_info.get_op_assignment_info() { let mut lock_guard = target.write_lock::().unwrap(); let args = &mut [&mut *lock_guard, &mut new_val]; if self.fast_operators() { if let Some((func, need_context)) = - get_builtin_op_assignment_fn(op_assign, args[0], args[1]) + get_builtin_op_assignment_fn(op_x, args[0], args[1]) { // Built-in found auto_restore! { let orig_level = global.level; global.level += 1 } let context = if need_context { - let op = op_assign.literal_syntax(); let source = global.source(); - Some((self, op, source, &*global, pos).into()) + Some((self, op_x_str, source, &*global, pos).into()) } else { None }; @@ -149,20 +148,17 @@ impl Engine { } } - let token = Some(op_assign); - let op_assign = op_assign.literal_syntax(); + let opx = Some(op_x); - match self.exec_native_fn_call(global, caches, op_assign, token, hash1, args, true, pos) - { + match self.exec_native_fn_call(global, caches, op_x_str, opx, hash_x, args, true, pos) { Ok(_) => (), - Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) => + Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_x_str)) => { // Expand to `var = var op rhs` - let token = Some(op); - let op = op.literal_syntax(); + let op = Some(op); *args[0] = self - .exec_native_fn_call(global, caches, op, token, hash2, args, true, pos)? + .exec_native_fn_call(global, caches, op_str, op, hash, args, true, pos)? .0; } Err(err) => return Err(err), diff --git a/src/func/call.rs b/src/func/call.rs index 2eb7ace7..2abda94b 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -1491,7 +1491,7 @@ impl Engine { } else { format!( "{namespace}{}{}", - crate::tokenizer::Token::DoubleColon.literal_syntax(), + crate::engine::NAMESPACE_SEPARATOR, self.gen_fn_call_signature(fn_name, args) ) }; diff --git a/src/optimizer.rs b/src/optimizer.rs index 6de5862f..d01f98c3 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -5,7 +5,9 @@ use crate::ast::{ ASTFlags, Expr, FlowControl, OpAssignment, Stmt, StmtBlock, StmtBlockContainer, SwitchCasesCollection, }; -use crate::engine::{KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_PRINT, KEYWORD_TYPE_OF}; +use crate::engine::{ + KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_PRINT, KEYWORD_TYPE_OF, OP_NOT, +}; use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::builtin::get_builtin_binary_op_fn; use crate::func::hashing::get_hasher; @@ -25,9 +27,6 @@ use std::{ mem, }; -/// Standard not operator. -const OP_NOT: &str = Token::Bang.literal_syntax(); - /// Level of optimization performed. #[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] #[non_exhaustive] diff --git a/src/packages/lang_core.rs b/src/packages/lang_core.rs index ce75f7ef..229113d3 100644 --- a/src/packages/lang_core.rs +++ b/src/packages/lang_core.rs @@ -287,7 +287,8 @@ fn collect_fn_metadata( #[cfg(not(feature = "no_module"))] { - use crate::{tokenizer::Token::DoubleColon, Shared, SmartString}; + use crate::engine::NAMESPACE_SEPARATOR; + use crate::{Shared, SmartString}; // Recursively scan modules for script-defined functions. fn scan_module( @@ -305,7 +306,7 @@ fn collect_fn_metadata( use std::fmt::Write; let mut ns = SmartString::new_const(); - write!(&mut ns, "{namespace}{}{name}", DoubleColon.literal_syntax()).unwrap(); + write!(&mut ns, "{namespace}{}{name}", NAMESPACE_SEPARATOR).unwrap(); scan_module(engine, list, &ns, m, filter); } } diff --git a/src/parser.rs b/src/parser.rs index 0f452d8c..ac296d15 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,7 +7,7 @@ use crate::ast::{ FnCallHashes, Ident, Namespace, OpAssignment, RangeCase, ScriptFnDef, Stmt, StmtBlock, StmtBlockContainer, SwitchCasesCollection, }; -use crate::engine::{Precedence, KEYWORD_THIS, OP_CONTAINS}; +use crate::engine::{Precedence, KEYWORD_THIS, OP_CONTAINS, OP_NOT}; use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::{hashing::get_hasher, StraightHashMap}; use crate::tokenizer::{ @@ -1299,6 +1299,10 @@ impl Engine { } } + expressions.shrink_to_fit(); + cases.shrink_to_fit(); + ranges.shrink_to_fit(); + let cases = SwitchCasesCollection { expressions, cases, @@ -2418,14 +2422,13 @@ impl Engine { fn_call } else { // Put a `!` call in front - let op = Token::Bang.literal_syntax(); let mut args = FnArgsVec::new_const(); args.push(fn_call); let not_base = FnCallExpr { namespace: Namespace::NONE, - name: state.get_interned_string(op), - hashes: FnCallHashes::from_native_only(calc_fn_hash(None, op, 1)), + name: state.get_interned_string(OP_NOT), + hashes: FnCallHashes::from_native_only(calc_fn_hash(None, OP_NOT, 1)), args, op_token: Some(Token::Bang), capture_parent_scope: false, From 10089c5cb0ff4e4b11cb56e709588e33df46fed7 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 12 Feb 2023 23:20:14 +0800 Subject: [PATCH 05/41] Support switch range cases for floating-point values. --- CHANGELOG.md | 1 + src/ast/stmt.rs | 56 ++++++++++++++++++++++++++++++++++++++------ src/eval/stmt.rs | 17 +++++++------- src/func/hashing.rs | 1 + src/optimizer.rs | 8 +++---- src/parser.rs | 26 ++++---------------- tests/expressions.rs | 4 ++-- 7 files changed, 68 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3126de3d..1e9732b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Enhancements ------------ * The functions `min` and `max` are added for numbers. +* Range cases in `switch` statements now also match floating-point and decimal values. In order to support this, however, small numeric ranges cases are no longer unrolled. Version 1.12.0 diff --git a/src/ast/stmt.rs b/src/ast/stmt.rs index cd26df7f..9c67bcab 100644 --- a/src/ast/stmt.rs +++ b/src/ast/stmt.rs @@ -4,8 +4,9 @@ use super::{ASTFlags, ASTNode, BinaryExpr, Expr, FnCallExpr, Ident}; use crate::engine::{KEYWORD_EVAL, OP_EQUALS}; use crate::func::StraightHashMap; use crate::tokenizer::Token; +use crate::types::dynamic::Union; use crate::types::Span; -use crate::{calc_fn_hash, Position, StaticVec, INT}; +use crate::{calc_fn_hash, Dynamic, Position, StaticVec, INT}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{ @@ -257,7 +258,7 @@ impl IntoIterator for RangeCase { type Item = INT; type IntoIter = Box>; - #[inline(always)] + #[inline] #[must_use] fn into_iter(self) -> Self::IntoIter { match self { @@ -269,7 +270,7 @@ impl IntoIterator for RangeCase { impl RangeCase { /// Returns `true` if the range contains no items. - #[inline(always)] + #[inline] #[must_use] pub fn is_empty(&self) -> bool { match self { @@ -278,7 +279,7 @@ impl RangeCase { } } /// Size of the range. - #[inline(always)] + #[inline] #[must_use] pub fn len(&self) -> INT { match self { @@ -288,15 +289,56 @@ impl RangeCase { Self::InclusiveInt(r, ..) => *r.end() - *r.start() + 1, } } - /// Is the specified number within this range? - #[inline(always)] + /// Is the specified value within this range? + #[inline] #[must_use] - pub fn contains(&self, n: INT) -> bool { + pub fn contains(&self, value: &Dynamic) -> bool { + match value { + Dynamic(Union::Int(v, ..)) => self.contains_int(*v), + #[cfg(not(feature = "no_float"))] + Dynamic(Union::Float(v, ..)) => self.contains_float(**v), + #[cfg(feature = "decimal")] + Dynamic(Union::Decimal(v, ..)) => self.contains_decimal(**v), + _ => false, + } + } + /// Is the specified number within this range? + #[inline] + #[must_use] + pub fn contains_int(&self, n: INT) -> bool { match self { Self::ExclusiveInt(r, ..) => r.contains(&n), Self::InclusiveInt(r, ..) => r.contains(&n), } } + /// Is the specified floating-point number within this range? + #[cfg(not(feature = "no_float"))] + #[inline] + #[must_use] + pub fn contains_float(&self, n: crate::FLOAT) -> bool { + use crate::FLOAT; + + match self { + Self::ExclusiveInt(r, ..) => ((r.start as FLOAT)..(r.end as FLOAT)).contains(&n), + Self::InclusiveInt(r, ..) => ((*r.start() as FLOAT)..=(*r.end() as FLOAT)).contains(&n), + } + } + /// Is the specified decimal number within this range? + #[cfg(feature = "decimal")] + #[inline] + #[must_use] + pub fn contains_decimal(&self, n: rust_decimal::Decimal) -> bool { + use rust_decimal::Decimal; + + match self { + Self::ExclusiveInt(r, ..) => { + (Into::::into(r.start)..Into::::into(r.end)).contains(&n) + } + Self::InclusiveInt(r, ..) => { + (Into::::into(*r.start())..=Into::::into(*r.end())).contains(&n) + } + } + } /// Is the specified range inclusive? #[inline(always)] #[must_use] diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index d6e5229e..194951c9 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -3,7 +3,8 @@ use super::{Caches, EvalContext, GlobalRuntimeState, Target}; use crate::api::events::VarDefInfo; use crate::ast::{ - ASTFlags, BinaryExpr, Expr, FlowControl, OpAssignment, Stmt, SwitchCasesCollection, + ASTFlags, BinaryExpr, ConditionalExpr, Expr, FlowControl, OpAssignment, Stmt, + SwitchCasesCollection, }; use crate::func::{get_builtin_op_assignment_fn, get_hasher}; use crate::types::dynamic::{AccessMode, Union}; @@ -359,15 +360,13 @@ impl Engine { break; } } - } else if value.is_int() && !ranges.is_empty() { + } else if !ranges.is_empty() { // Then check integer ranges - let value = value.as_int().expect("`INT`"); + for r in ranges.iter().filter(|r| r.contains(&value)) { + let ConditionalExpr { condition, expr } = &expressions[r.index()]; - for r in ranges.iter().filter(|r| r.contains(value)) { - let block = &expressions[r.index()]; - - let cond_result = match block.condition { - Expr::BoolConstant(b, ..) => b, + let cond_result = match condition { + Expr::BoolConstant(b, ..) => *b, ref c => self .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), c)? .as_bool() @@ -377,7 +376,7 @@ impl Engine { }; if cond_result { - result = Some(&block.expr); + result = Some(expr); break; } } diff --git a/src/func/hashing.rs b/src/func/hashing.rs index ac0f1b27..58c4202b 100644 --- a/src/func/hashing.rs +++ b/src/func/hashing.rs @@ -28,6 +28,7 @@ impl Hasher for StraightHasher { self.0 } #[cold] + #[inline(never)] fn write(&mut self, _bytes: &[u8]) { panic!("StraightHasher can only hash u64 values"); } diff --git a/src/optimizer.rs b/src/optimizer.rs index d01f98c3..a502a6f4 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -563,16 +563,14 @@ fn optimize_stmt(stmt: &mut Stmt, state: &mut OptimizerState, preserve_result: b } // Then check ranges - if value.is_int() && !ranges.is_empty() { - let value = value.as_int().unwrap(); - + if !ranges.is_empty() { // Only one range or all ranges without conditions if ranges.len() == 1 || ranges .iter() .all(|r| expressions[r.index()].is_always_true()) { - if let Some(r) = ranges.iter().find(|r| r.contains(value)) { + if let Some(r) = ranges.iter().find(|r| r.contains(&value)) { let range_block = &mut expressions[r.index()]; if range_block.is_always_true() { @@ -619,7 +617,7 @@ fn optimize_stmt(stmt: &mut Stmt, state: &mut OptimizerState, preserve_result: b let old_ranges_len = ranges.len(); - ranges.retain(|r| r.contains(value)); + ranges.retain(|r| r.contains(&value)); if ranges.len() != old_ranges_len { state.set_dirty(); diff --git a/src/parser.rs b/src/parser.rs index ac296d15..67807e87 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -19,7 +19,7 @@ use crate::types::StringsInterner; use crate::{ calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, ExclusiveRange, FnArgsVec, Identifier, ImmutableString, InclusiveRange, LexError, OptimizationLevel, ParseError, Position, - Scope, Shared, SmartString, StaticVec, AST, INT, PERR, + Scope, Shared, SmartString, StaticVec, AST, PERR, }; use bitflags::bitflags; #[cfg(feature = "no_std")] @@ -42,9 +42,6 @@ const SCOPE_SEARCH_BARRIER_MARKER: &str = "$ BARRIER $"; /// The message: `TokenStream` never ends const NEVER_ENDS: &str = "`Token`"; -/// Unroll `switch` ranges no larger than this. -const SMALL_SWITCH_RANGE: INT = 16; - /// _(internals)_ A type that encapsulates the current state of the parser. /// Exported under the `internals` feature only. pub struct ParseState<'e, 's> { @@ -1216,7 +1213,6 @@ impl Engine { let stmt_block: StmtBlock = stmt.into(); (Expr::Stmt(stmt_block.into()), need_comma) }; - let has_condition = !matches!(condition, Expr::BoolConstant(true, ..)); expressions.push((condition, action_expr).into()); let index = expressions.len() - 1; @@ -1240,23 +1236,9 @@ impl Engine { if let Some(mut r) = range_value { if !r.is_empty() { - // Do not unroll ranges if there are previous non-unrolled ranges - if !has_condition && ranges.is_empty() && r.len() <= SMALL_SWITCH_RANGE - { - // Unroll small range - r.into_iter().for_each(|n| { - let hasher = &mut get_hasher(); - Dynamic::from_int(n).hash(hasher); - cases - .entry(hasher.finish()) - .and_modify(|cases| cases.push(index)) - .or_insert_with(|| [index].into()); - }); - } else { - // Other range - r.set_index(index); - ranges.push(r); - } + // Other range + r.set_index(index); + ranges.push(r); } continue; } diff --git a/tests/expressions.rs b/tests/expressions.rs index 182f9fa9..e15008e8 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -50,8 +50,8 @@ fn test_expressions() -> Result<(), Box> { " switch x { 0 => 1, - 1..10 => 123, 10 => 42, + 1..10 => 123, } " )?, @@ -63,11 +63,11 @@ fn test_expressions() -> Result<(), Box> { " switch x { 0 => 1, + 10 => 42, 1..10 => { let y = 123; y } - 10 => 42, } " ) From 7da20dd090ae03aba68a8025e23d432d65b04204 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 13 Feb 2023 08:58:59 +0800 Subject: [PATCH 06/41] Forbid floating-point switch cases after range case. --- src/parser.rs | 17 ++++++++++++++--- src/types/parse_error.rs | 4 ++-- tests/switch.rs | 27 +++++++++++++++------------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 67807e87..fc9e8966 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -14,7 +14,7 @@ use crate::tokenizer::{ is_keyword_function, is_valid_function_name, is_valid_identifier, Token, TokenStream, TokenizerControl, }; -use crate::types::dynamic::AccessMode; +use crate::types::dynamic::{AccessMode, Union}; use crate::types::StringsInterner; use crate::{ calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, ExclusiveRange, FnArgsVec, @@ -1243,8 +1243,19 @@ impl Engine { continue; } - if value.is_int() && !ranges.is_empty() { - return Err(PERR::WrongSwitchIntegerCase.into_err(expr.start_position())); + if !ranges.is_empty() { + let forbidden = match value { + Dynamic(Union::Int(..)) => true, + #[cfg(not(feature = "no_float"))] + Dynamic(Union::Float(..)) => true, + #[cfg(feature = "decimal")] + Dynamic(Union::Decimal(..)) => true, + _ => false, + }; + + if forbidden { + return Err(PERR::WrongSwitchIntegerCase.into_err(expr.start_position())); + } } let hasher = &mut get_hasher(); diff --git a/src/types/parse_error.rs b/src/types/parse_error.rs index 19156ef8..e6062d5d 100644 --- a/src/types/parse_error.rs +++ b/src/types/parse_error.rs @@ -105,7 +105,7 @@ pub enum ParseErrorType { DuplicatedSwitchCase, /// A variable name is duplicated. Wrapped value is the variable name. DuplicatedVariable(String), - /// An integer case of a `switch` statement is in an appropriate place. + /// A numeric case of a `switch` statement is in an appropriate place. WrongSwitchIntegerCase, /// The default case of a `switch` statement is in an appropriate place. WrongSwitchDefaultCase, @@ -236,7 +236,7 @@ impl fmt::Display for ParseErrorType { Self::Reserved(s) if is_valid_identifier(s.as_str()) => write!(f, "'{s}' is a reserved keyword"), Self::Reserved(s) => write!(f, "'{s}' is a reserved symbol"), Self::UnexpectedEOF => f.write_str("Script is incomplete"), - Self::WrongSwitchIntegerCase => f.write_str("Integer switch case cannot follow a range case"), + Self::WrongSwitchIntegerCase => f.write_str("Numeric switch case cannot follow a range case"), Self::WrongSwitchDefaultCase => f.write_str("Default switch case must be the last"), Self::WrongSwitchCaseCondition => f.write_str("This switch case cannot have a condition"), Self::PropertyExpected => f.write_str("Expecting name of a property"), diff --git a/tests/switch.rs b/tests/switch.rs index 0de962f7..f37dacc0 100644 --- a/tests/switch.rs +++ b/tests/switch.rs @@ -10,9 +10,7 @@ fn test_switch() -> Result<(), Box> { engine.eval::("switch 2 { 1 => (), 2 => 'a', 42 => true }")?, 'a' ); - engine - .run("switch 3 { 1 => (), 2 => 'a', 42 => true }") - .unwrap(); + engine.run("switch 3 { 1 => (), 2 => 'a', 42 => true }")?; assert_eq!( engine.eval::("switch 3 { 1 => (), 2 => 'a', 42 => true, _ => 123 }")?, 123 @@ -31,15 +29,13 @@ fn test_switch() -> Result<(), Box> { )?, 'a' ); - assert!(engine - .eval_with_scope::(&mut scope, "switch x { 1 => (), 2 => 'a', 42 => true }") - .unwrap()); - assert!(engine - .eval_with_scope::(&mut scope, "switch x { 1 => (), 2 => 'a', _ => true }") - .unwrap()); - let _: () = engine - .eval_with_scope::<()>(&mut scope, "switch x { 1 => 123, 2 => 'a' }") - .unwrap(); + assert!( + engine.eval_with_scope::(&mut scope, "switch x { 1 => (), 2 => 'a', 42 => true }")? + ); + assert!( + engine.eval_with_scope::(&mut scope, "switch x { 1 => (), 2 => 'a', _ => true }")? + ); + let _: () = engine.eval_with_scope::<()>(&mut scope, "switch x { 1 => 123, 2 => 'a' }")?; assert_eq!( engine.eval_with_scope::( @@ -276,6 +272,13 @@ fn test_switch_ranges() -> Result<(), Box> { ).expect_err("should error").err_type(), ParseErrorType::WrongSwitchIntegerCase )); + #[cfg(not(feature = "no_float"))] + assert!(matches!( + engine.compile( + "switch x { 10..20 => (), 20..=42 => 'a', 25..45 => 'z', 42.0 => 'x', 30..100 => true }" + ).expect_err("should error").err_type(), + ParseErrorType::WrongSwitchIntegerCase + )); assert_eq!( engine.eval_with_scope::( &mut scope, From 7fc72e8c28d4b2e247b678934ae5d3c914d44f07 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 13 Feb 2023 08:59:58 +0800 Subject: [PATCH 07/41] Use ? in tests. --- tests/arrays.rs | 18 +++++++----------- tests/bool_op.rs | 28 ++++++++++++---------------- tests/closures.rs | 16 ++++++++-------- tests/comments.rs | 2 +- tests/compound_equality.rs | 10 +++++----- tests/float.rs | 4 +--- tests/get_set.rs | 8 +++----- tests/not.rs | 8 +++----- tests/unit.rs | 4 +--- 9 files changed, 41 insertions(+), 57 deletions(-) diff --git a/tests/arrays.rs b/tests/arrays.rs index a363873f..8820a54a 100644 --- a/tests/arrays.rs +++ b/tests/arrays.rs @@ -515,14 +515,12 @@ fn test_arrays_map_reduce() -> Result<(), Box> { 3 ); - engine - .eval::<()>( - " + engine.eval::<()>( + " let x = [1, 2, 3, 2, 1]; x.find(|v| v > 4) ", - ) - .unwrap(); + )?; assert_eq!( engine.eval::( @@ -534,14 +532,12 @@ fn test_arrays_map_reduce() -> Result<(), Box> { 2 ); - engine - .eval::<()>( - " + engine.eval::<()>( + " let x = [#{alice: 1}, #{bob: 2}, #{clara: 3}]; x.find_map(|v| v.dave) ", - ) - .unwrap(); + )?; Ok(()) } @@ -550,7 +546,7 @@ fn test_arrays_map_reduce() -> Result<(), Box> { fn test_arrays_elvis() -> Result<(), Box> { let engine = Engine::new(); - engine.eval::<()>("let x = (); x?[2]").unwrap(); + engine.eval::<()>("let x = (); x?[2]")?; engine.run("let x = (); x?[2] = 42")?; diff --git a/tests/bool_op.rs b/tests/bool_op.rs index 30a551cd..247fc33b 100644 --- a/tests/bool_op.rs +++ b/tests/bool_op.rs @@ -4,8 +4,8 @@ use rhai::{Engine, EvalAltResult}; fn test_bool_op1() -> Result<(), Box> { let engine = Engine::new(); - assert!(engine.eval::("true && (false || true)").unwrap()); - assert!(engine.eval::("true & (false | true)").unwrap()); + assert!(engine.eval::("true && (false || true)")?); + assert!(engine.eval::("true & (false | true)")?); Ok(()) } @@ -14,8 +14,8 @@ fn test_bool_op1() -> Result<(), Box> { fn test_bool_op2() -> Result<(), Box> { let engine = Engine::new(); - assert!(!engine.eval::("false && (false || true)").unwrap()); - assert!(!engine.eval::("false & (false | true)").unwrap()); + assert!(!engine.eval::("false && (false || true)")?); + assert!(!engine.eval::("false & (false | true)")?); Ok(()) } @@ -25,9 +25,9 @@ fn test_bool_op3() -> Result<(), Box> { let engine = Engine::new(); assert!(engine.eval::("true && (false || 123)").is_err()); - assert!(engine.eval::("true && (true || { throw })").unwrap()); + assert!(engine.eval::("true && (true || { throw })")?); assert!(engine.eval::("123 && (false || true)").is_err()); - assert!(!engine.eval::("false && (true || { throw })").unwrap()); + assert!(!engine.eval::("false && (true || { throw })")?); Ok(()) } @@ -36,23 +36,19 @@ fn test_bool_op3() -> Result<(), Box> { fn test_bool_op_short_circuit() -> Result<(), Box> { let engine = Engine::new(); - assert!(engine - .eval::( - " + assert!(engine.eval::( + " let x = true; x || { throw; }; " - ) - .unwrap()); + )?); - assert!(!engine - .eval::( - " + assert!(!engine.eval::( + " let x = false; x && { throw; }; " - ) - .unwrap()); + )?); Ok(()) } diff --git a/tests/closures.rs b/tests/closures.rs index d2576e80..0e934c30 100644 --- a/tests/closures.rs +++ b/tests/closures.rs @@ -369,9 +369,9 @@ fn test_closures_external() -> Result<(), Box> { let fn_ptr = engine.eval_ast::(&ast)?; - let f = move |x: INT| -> String { fn_ptr.call(&engine, &ast, (x,)).unwrap() }; + let f = move |x: INT| fn_ptr.call::(&engine, &ast, (x,)); - assert_eq!(f(42), "hello42"); + assert_eq!(f(42)?, "hello42"); Ok(()) } @@ -383,20 +383,20 @@ fn test_closures_callback() -> Result<(), Box> { type SingleNode = Rc; trait Node { - fn run(&self, x: INT) -> INT; + fn run(&self, x: INT) -> Result>; } struct PhaserNode { - func: Box INT>, + func: Box Result>>, } impl Node for PhaserNode { - fn run(&self, x: INT) -> INT { + fn run(&self, x: INT) -> Result> { (self.func)(x) } } - fn phaser(callback: impl Fn(INT) -> INT + 'static) -> impl Node { + fn phaser(callback: impl Fn(INT) -> Result> + 'static) -> impl Node { PhaserNode { func: Box::new(callback), } @@ -419,7 +419,7 @@ fn test_closures_callback() -> Result<(), Box> { let engine = engine2.clone(); let ast = ast2.clone(); - let callback = Box::new(move |x: INT| fp.call(&engine.borrow(), &ast, (x,)).unwrap()); + let callback = Box::new(move |x: INT| fp.call(&engine.borrow(), &ast, (x,))); Rc::new(phaser(callback)) as SingleNode }); @@ -428,7 +428,7 @@ fn test_closures_callback() -> Result<(), Box> { let cb = shared_engine.borrow().eval_ast::(&ast)?; - assert_eq!(cb.run(21), 42); + assert_eq!(cb.run(21)?, 42); Ok(()) } diff --git a/tests/comments.rs b/tests/comments.rs index 15f15afc..9d981c59 100644 --- a/tests/comments.rs +++ b/tests/comments.rs @@ -21,7 +21,7 @@ fn test_comments() -> Result<(), Box> { 42 ); - engine.run("/* Hello world */").unwrap(); + engine.run("/* Hello world */")?; Ok(()) } diff --git a/tests/compound_equality.rs b/tests/compound_equality.rs index e7de4cec..bd93f2cf 100644 --- a/tests/compound_equality.rs +++ b/tests/compound_equality.rs @@ -5,8 +5,8 @@ fn test_or_equals() -> Result<(), Box> { let engine = Engine::new(); assert_eq!(engine.eval::("let x = 16; x |= 74; x")?, 90); - assert!(engine.eval::("let x = true; x |= false; x").unwrap()); - assert!(engine.eval::("let x = false; x |= true; x").unwrap()); + assert!(engine.eval::("let x = true; x |= false; x")?); + assert!(engine.eval::("let x = false; x |= true; x")?); Ok(()) } @@ -16,9 +16,9 @@ fn test_and_equals() -> Result<(), Box> { let engine = Engine::new(); assert_eq!(engine.eval::("let x = 16; x &= 31; x")?, 16); - assert!(!engine.eval::("let x = true; x &= false; x").unwrap()); - assert!(!engine.eval::("let x = false; x &= true; x").unwrap()); - assert!(engine.eval::("let x = true; x &= true; x").unwrap()); + assert!(!engine.eval::("let x = true; x &= false; x")?); + assert!(!engine.eval::("let x = false; x &= true; x")?); + assert!(engine.eval::("let x = true; x &= true; x")?); Ok(()) } diff --git a/tests/float.rs b/tests/float.rs index b09740e6..a44243ed 100644 --- a/tests/float.rs +++ b/tests/float.rs @@ -9,9 +9,7 @@ fn test_float() -> Result<(), Box> { assert!(engine.eval::("let x = 0.0; let y = 1.0; x < y")?); assert!(!engine.eval::("let x = 0.0; let y = 1.0; x > y")?); - assert!(!engine - .eval::("let x = 0.; let y = 1.; x > y") - .unwrap()); + assert!(!engine.eval::("let x = 0.; let y = 1.; x > y")?); assert!((engine.eval::("let x = 9.9999; x")? - 9.9999 as FLOAT).abs() < EPSILON); Ok(()) diff --git a/tests/get_set.rs b/tests/get_set.rs index d5af9080..62e57805 100644 --- a/tests/get_set.rs +++ b/tests/get_set.rs @@ -399,11 +399,9 @@ fn test_get_set_indexer() -> Result<(), Box> { fn test_get_set_elvis() -> Result<(), Box> { let engine = Engine::new(); - engine.eval::<()>("let x = (); x?.foo.bar.baz").unwrap(); - engine.eval::<()>("let x = (); x?.foo(1,2,3)").unwrap(); - engine - .eval::<()>("let x = #{a:()}; x.a?.foo.bar.baz") - .unwrap(); + engine.eval::<()>("let x = (); x?.foo.bar.baz")?; + engine.eval::<()>("let x = (); x?.foo(1,2,3)")?; + engine.eval::<()>("let x = #{a:()}; x.a?.foo.bar.baz")?; assert_eq!(engine.eval::("let x = 'x'; x?.type_of()")?, "char"); Ok(()) diff --git a/tests/not.rs b/tests/not.rs index c9db6e2f..7440ec48 100644 --- a/tests/not.rs +++ b/tests/not.rs @@ -4,14 +4,12 @@ use rhai::{Engine, EvalAltResult}; fn test_not() -> Result<(), Box> { let engine = Engine::new(); - assert!(!engine - .eval::("let not_true = !true; not_true") - .unwrap()); + assert!(!engine.eval::("let not_true = !true; not_true")?); #[cfg(not(feature = "no_function"))] - assert!(engine.eval::("fn not(x) { !x } not(false)").unwrap()); + assert!(engine.eval::("fn not(x) { !x } not(false)")?); - assert!(engine.eval::("!!!!true").unwrap()); + assert!(engine.eval::("!!!!true")?); Ok(()) } diff --git a/tests/unit.rs b/tests/unit.rs index ed3d2847..d2abd3c2 100644 --- a/tests/unit.rs +++ b/tests/unit.rs @@ -10,9 +10,7 @@ fn test_unit() -> Result<(), Box> { #[test] fn test_unit_eq() -> Result<(), Box> { let engine = Engine::new(); - assert!(engine - .eval::("let x = (); let y = (); x == y") - .unwrap()); + assert!(engine.eval::("let x = (); let y = (); x == y")?); Ok(()) } From 513a1ab4350cd5abee865dc93030e255e446eafb Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 13 Feb 2023 21:12:17 +0800 Subject: [PATCH 08/41] Allow access to scope by loaded module. --- CHANGELOG.md | 6 ++++++ src/eval/stmt.rs | 12 +++++++----- src/module/mod.rs | 14 ++++++++------ src/module/resolvers/file.rs | 24 +++++++++++------------- src/module/resolvers/mod.rs | 6 ++++-- src/types/scope.rs | 17 ++++++++++++++--- 6 files changed, 50 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9732b6..7a2a0515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ Bug fixes * `x += y` where `x` and `y` are `char` now works correctly. * Expressions such as `!inside` now parses correctly instead of as `!in` followed by `side`. +Potentially breaking changes +---------------------------- + +* The trait method `ModuleResolver::resolve_raw` (which is a low-level API) now takes a `&mut Scope` parameter. This is a breaking change because the signature is modified, but this trait method has a default and is rarely called/implemented in practice. +* `Module::eval_ast_as_new_raw` (a low-level API) now takes a `&mut Scope` instead of the `Scope` parameter. This is a breaking change because the `&mut` is now required. + Enhancements ------------ diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 194951c9..33a26958 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -802,14 +802,16 @@ impl Engine { let module = resolver .as_ref() - .and_then(|r| match r.resolve_raw(self, global, &path, path_pos) { - Err(err) if matches!(*err, ERR::ErrorModuleNotFound(..)) => None, - result => Some(result), - }) + .and_then( + |r| match r.resolve_raw(self, global, scope, &path, path_pos) { + Err(err) if matches!(*err, ERR::ErrorModuleNotFound(..)) => None, + result => Some(result), + }, + ) .or_else(|| { Some( self.module_resolver - .resolve_raw(self, global, &path, path_pos), + .resolve_raw(self, global, scope, &path, path_pos), ) }) .unwrap_or_else(|| { diff --git a/src/module/mod.rs b/src/module/mod.rs index 87b4f831..0bed49c0 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -2083,9 +2083,10 @@ impl Module { ast: &crate::AST, engine: &crate::Engine, ) -> RhaiResultOf { + let mut scope = scope; let global = &mut crate::eval::GlobalRuntimeState::new(engine); - Self::eval_ast_as_new_raw(engine, scope, global, ast) + Self::eval_ast_as_new_raw(engine, &mut scope, global, ast) } /// Create a new [`Module`] by evaluating an [`AST`][crate::AST]. /// @@ -2101,13 +2102,12 @@ impl Module { #[cfg(not(feature = "no_module"))] pub fn eval_ast_as_new_raw( engine: &crate::Engine, - scope: crate::Scope, + scope: &mut crate::Scope, global: &mut crate::eval::GlobalRuntimeState, ast: &crate::AST, ) -> RhaiResultOf { - let mut scope = scope; - // Save global state + let orig_scope_len = scope.len(); let orig_imports_len = global.num_imports(); let orig_source = global.source.clone(); @@ -2120,7 +2120,7 @@ impl Module { // Run the script let caches = &mut crate::eval::Caches::new(); - let result = engine.eval_ast_with_scope_raw(global, caches, &mut scope, ast); + let result = engine.eval_ast_with_scope_raw(global, caches, scope, ast); // Create new module let mut module = Module::new(); @@ -2162,7 +2162,9 @@ impl Module { }); // Variables with an alias left in the scope become module variables - for (_name, mut value, mut aliases) in scope { + while scope.len() > orig_scope_len { + let (_name, mut value, mut aliases) = scope.pop_entry().expect("not empty"); + value.deep_scan(|v| { if let Some(fn_ptr) = v.downcast_mut::() { fn_ptr.set_encapsulated_environ(Some(environ.clone())); diff --git a/src/module/resolvers/file.rs b/src/module/resolvers/file.rs index 33a99286..ea62c5fb 100644 --- a/src/module/resolvers/file.rs +++ b/src/module/resolvers/file.rs @@ -290,15 +290,15 @@ impl FileModuleResolver { fn impl_resolve( &self, engine: &Engine, - global: Option<&mut GlobalRuntimeState>, + global: &mut GlobalRuntimeState, + scope: &mut Scope, source: Option<&str>, path: &str, pos: Position, ) -> Result> { // Load relative paths from source if there is no base path specified let source_path = global - .as_ref() - .and_then(|g| g.source()) + .source() .or(source) .and_then(|p| Path::new(p).parent()); @@ -321,14 +321,9 @@ impl FileModuleResolver { ast.set_source(path); - let scope = Scope::new(); - - let m: Shared<_> = match global { - Some(global) => Module::eval_ast_as_new_raw(engine, scope, global, &ast), - None => Module::eval_ast_as_new(scope, &ast, engine), - } - .map_err(|err| Box::new(ERR::ErrorInModule(path.to_string(), err, pos)))? - .into(); + let m: Shared<_> = Module::eval_ast_as_new_raw(engine, scope, global, &ast) + .map_err(|err| Box::new(ERR::ErrorInModule(path.to_string(), err, pos)))? + .into(); if self.is_cache_enabled() { locked_write(&self.cache).insert(file_path, m.clone()); @@ -343,10 +338,11 @@ impl ModuleResolver for FileModuleResolver { &self, engine: &Engine, global: &mut GlobalRuntimeState, + scope: &mut Scope, path: &str, pos: Position, ) -> RhaiResultOf { - self.impl_resolve(engine, Some(global), None, path, pos) + self.impl_resolve(engine, global, scope, None, path, pos) } #[inline(always)] @@ -357,7 +353,9 @@ impl ModuleResolver for FileModuleResolver { path: &str, pos: Position, ) -> RhaiResultOf { - self.impl_resolve(engine, None, source, path, pos) + let global = &mut GlobalRuntimeState::new(engine); + let scope = &mut Scope::new(); + self.impl_resolve(engine, global, scope, source, path, pos) } /// Resolve an `AST` based on a path string. diff --git a/src/module/resolvers/mod.rs b/src/module/resolvers/mod.rs index 6316a845..7e70245f 100644 --- a/src/module/resolvers/mod.rs +++ b/src/module/resolvers/mod.rs @@ -1,6 +1,6 @@ use crate::eval::GlobalRuntimeState; use crate::func::SendSync; -use crate::{Engine, Position, RhaiResultOf, SharedModule, AST}; +use crate::{Engine, Position, RhaiResultOf, Scope, SharedModule, AST}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -27,15 +27,17 @@ pub trait ModuleResolver: SendSync { pos: Position, ) -> RhaiResultOf; - /// Resolve a module based on a path string, given a [`GlobalRuntimeState`]. + /// Resolve a module based on a path string, given a [`GlobalRuntimeState`] and the current [`Scope`]. /// /// # WARNING - Low Level API /// /// This function is very low level. + #[allow(unused_variables)] fn resolve_raw( &self, engine: &Engine, global: &mut GlobalRuntimeState, + scope: &mut Scope, path: &str, pos: Position, ) -> RhaiResultOf { diff --git a/src/types/scope.rs b/src/types/scope.rs index 712298fe..927225b0 100644 --- a/src/types/scope.rs +++ b/src/types/scope.rs @@ -385,11 +385,22 @@ impl Scope<'_> { /// ``` #[inline(always)] pub fn pop(&mut self) -> &mut Self { - self.names.pop().expect("`Scope` must not be empty"); - let _ = self.values.pop().expect("`Scope` must not be empty"); - self.aliases.pop().expect("`Scope` must not be empty"); + self.names.pop().expect("not empty"); + let _ = self.values.pop().expect("not empty"); + self.aliases.pop().expect("not empty"); self } + /// Remove the last entry from the [`Scope`] and return it. + #[inline(always)] + pub(crate) fn pop_entry(&mut self) -> Option<(Identifier, Dynamic, Vec)> { + self.values.pop().map(|value| { + ( + self.names.pop().expect("not empty"), + value, + self.aliases.pop().expect("not empty"), + ) + }) + } /// Truncate (rewind) the [`Scope`] to a previous size. /// /// # Example From 6b301b4e703bf5d6e8cdf2ed277313fba25d217a Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 16 Feb 2023 20:35:15 +0800 Subject: [PATCH 09/41] Set allow_loop_expressions to true by default. --- CHANGELOG.md | 2 ++ src/api/options.rs | 1 + src/parser.rs | 4 +++- tests/looping.rs | 42 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2a0515..64e02268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,12 +18,14 @@ Potentially breaking changes * The trait method `ModuleResolver::resolve_raw` (which is a low-level API) now takes a `&mut Scope` parameter. This is a breaking change because the signature is modified, but this trait method has a default and is rarely called/implemented in practice. * `Module::eval_ast_as_new_raw` (a low-level API) now takes a `&mut Scope` instead of the `Scope` parameter. This is a breaking change because the `&mut` is now required. +* `Engine::allow_loop_expressions` now correctly defaults to `true` (was erroneously `false` by default). Enhancements ------------ * The functions `min` and `max` are added for numbers. * Range cases in `switch` statements now also match floating-point and decimal values. In order to support this, however, small numeric ranges cases are no longer unrolled. +* Loading a module via `import` now gives the module access to the current scope, including variables and constants defined inside. Version 1.12.0 diff --git a/src/api/options.rs b/src/api/options.rs index 325dbbcb..057a9ac3 100644 --- a/src/api/options.rs +++ b/src/api/options.rs @@ -41,6 +41,7 @@ impl LangOptions { pub fn new() -> Self { Self::IF_EXPR | Self::SWITCH_EXPR + | Self::LOOP_EXPR | Self::STMT_EXPR | Self::LOOPING | Self::SHADOWING diff --git a/src/parser.rs b/src/parser.rs index fc9e8966..db75c099 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1254,7 +1254,9 @@ impl Engine { }; if forbidden { - return Err(PERR::WrongSwitchIntegerCase.into_err(expr.start_position())); + return Err( + PERR::WrongSwitchIntegerCase.into_err(expr.start_position()) + ); } } diff --git a/tests/looping.rs b/tests/looping.rs index a27a3555..e011eb6b 100644 --- a/tests/looping.rs +++ b/tests/looping.rs @@ -20,7 +20,7 @@ fn test_loop() -> Result<(), Box> { } } - return x; + x " )?, 21 @@ -53,3 +53,43 @@ fn test_loop() -> Result<(), Box> { Ok(()) } + +#[test] +fn test_loop_expression() -> Result<(), Box> { + let mut engine = Engine::new(); + + assert_eq!( + engine.eval::( + " + let x = 0; + + let value = while x < 10 { + if x % 5 == 0 { break 42; } + x += 1; + }; + + value + " + )?, + 42 + ); + + engine.set_allow_loop_expressions(false); + + assert!(engine + .eval::( + " + let x = 0; + + let value = while x < 10 { + if x % 5 == 0 { break 42; } + x += 1; + }; + + value + " + ) + .is_err()); + + Ok(()) +} From 6d888cc0e4f0ae49e0bc794de92e7865cc583b4c Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 18 Feb 2023 14:28:38 +0800 Subject: [PATCH 10/41] Fix bug. --- src/parser.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index db75c099..f4fd7ad5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1397,20 +1397,26 @@ impl Engine { .into(), )), // Loops are allowed to act as expressions - Token::While | Token::Loop if settings.has_option(LangOptions::LOOP_EXPR) => { + Token::While | Token::Loop + if self.allow_looping() && settings.has_option(LangOptions::LOOP_EXPR) => + { Expr::Stmt(Box::new( self.parse_while_loop(input, state, lib, settings.level_up()?)? .into(), )) } - Token::Do if settings.has_option(LangOptions::LOOP_EXPR) => Expr::Stmt(Box::new( - self.parse_do(input, state, lib, settings.level_up()?)? - .into(), - )), - Token::For if settings.has_option(LangOptions::LOOP_EXPR) => Expr::Stmt(Box::new( - self.parse_for(input, state, lib, settings.level_up()?)? - .into(), - )), + Token::Do if self.allow_looping() && settings.has_option(LangOptions::LOOP_EXPR) => { + Expr::Stmt(Box::new( + self.parse_do(input, state, lib, settings.level_up()?)? + .into(), + )) + } + Token::For if self.allow_looping() && settings.has_option(LangOptions::LOOP_EXPR) => { + Expr::Stmt(Box::new( + self.parse_for(input, state, lib, settings.level_up()?)? + .into(), + )) + } // Switch statement is allowed to act as expressions Token::Switch if settings.has_option(LangOptions::SWITCH_EXPR) => Expr::Stmt(Box::new( self.parse_switch(input, state, lib, settings.level_up()?)? From 426055d4d3d98ffb7b0755e5ac135350ed28a793 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 20 Feb 2023 10:06:26 +0800 Subject: [PATCH 11/41] Update rustyline. --- Cargo.toml | 4 ++-- src/bin/rhai-repl.rs | 31 +++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 90dba115..06c4ec6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"], unicode-xid = { version = "0.2", default-features = false, optional = true } rust_decimal = { version = "1.16", default-features = false, features = ["maths"], optional = true } getrandom = { version = "0.2", optional = true } -rustyline = { version = "10", optional = true } +rustyline = { version = "11", optional = true } document-features = { version = "0.2", optional = true } [dev-dependencies] @@ -151,4 +151,4 @@ features = ["document-features", "metadata", "serde", "internals", "decimal", "d [patch.crates-io] # Notice that a custom modified version of `rustyline` is used which supports bracketed paste on Windows. # This can be moved to the official version when bracketed paste is added. -rustyline = { git = "https://github.com/schungx/rustyline", branch = "v10" } +rustyline = { git = "https://github.com/schungx/rustyline", branch = "v11" } diff --git a/src/bin/rhai-repl.rs b/src/bin/rhai-repl.rs index afdff9dc..c296f381 100644 --- a/src/bin/rhai-repl.rs +++ b/src/bin/rhai-repl.rs @@ -2,7 +2,8 @@ use rhai::plugin::*; use rhai::{Dynamic, Engine, EvalAltResult, Module, Scope, AST, INT}; use rustyline::config::Builder; use rustyline::error::ReadlineError; -use rustyline::{Cmd, Editor, Event, EventHandler, KeyCode, KeyEvent, Modifiers, Movement}; +use rustyline::history::{History, SearchDirection}; +use rustyline::{Cmd, DefaultEditor, Event, EventHandler, KeyCode, KeyEvent, Modifiers, Movement}; use std::{env, fs::File, io::Read, path::Path, process::exit}; @@ -188,14 +189,14 @@ fn load_script_files(engine: &mut Engine) { } // Setup the Rustyline editor. -fn setup_editor() -> Editor<()> { +fn setup_editor() -> DefaultEditor { //env_logger::init(); let config = Builder::new() .tab_stop(4) .indent_size(4) .bracketed_paste(true) .build(); - let mut rl = Editor::<()>::with_config(config).unwrap(); + let mut rl = DefaultEditor::with_config(config).unwrap(); // Bind more keys @@ -336,7 +337,10 @@ fn main() { 'main_loop: loop { if let Some(replace) = replacement.take() { input = replace; - if rl.add_history_entry(input.clone()) { + if rl + .add_history_entry(input.clone()) + .expect("Failed to add history entry") + { history_offset += 1; } if input.contains('\n') { @@ -366,7 +370,9 @@ fn main() { if !cmd.is_empty() && !cmd.starts_with('!') && cmd.trim() != "history" - && rl.add_history_entry(input.clone()) + && rl + .add_history_entry(input.clone()) + .expect("Failed to add history entry") { history_offset += 1; } @@ -476,7 +482,7 @@ fn main() { let json = engine .gen_fn_metadata_with_ast_to_json(&main_ast, false) - .unwrap(); + .expect("Unable to generate JSON"); let mut f = std::fs::File::create("metadata.json") .expect("Unable to create `metadata.json`"); f.write_all(json.as_bytes()).expect("Unable to write data"); @@ -484,7 +490,7 @@ fn main() { continue; } "!!" => { - match rl.history().last() { + match rl.history().iter().last() { Some(line) => { replacement = Some(line.clone()); replacement_index = history_offset + rl.history().len() - 1; @@ -514,8 +520,12 @@ fn main() { _ if cmd.starts_with('!') => { if let Ok(num) = cmd[1..].parse::() { if num >= history_offset { - if let Some(line) = rl.history().get(num - history_offset) { - replacement = Some(line.clone()); + if let Some(line) = rl + .history() + .get(num - history_offset, SearchDirection::Forward) + .expect("Failed to get history entry") + { + replacement = Some(line.entry.into()); replacement_index = num; continue; } @@ -578,7 +588,8 @@ fn main() { main_ast.clear_statements(); } - rl.save_history(HISTORY_FILE).unwrap(); + rl.save_history(HISTORY_FILE) + .expect("Failed to save history"); println!("Bye!"); } From 00f2b07d388899ecb6d06acb418f2118c46cd4cf Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 20 Feb 2023 13:28:17 +0800 Subject: [PATCH 12/41] All symbols to start a custom syntax. --- CHANGELOG.md | 1 + src/api/custom_syntax.rs | 25 ++++++++++-------- src/parser.rs | 2 +- src/tokenizer.rs | 2 +- tests/custom_syntax.rs | 56 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64e02268..6be94fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Bug fixes * Syntax such as `foo.bar::baz` no longer panics, but returns a proper parse error. * `x += y` where `x` and `y` are `char` now works correctly. * Expressions such as `!inside` now parses correctly instead of as `!in` followed by `side`. +* Custom syntax starting with symbols now works correctly and no longer raises a parse error. Potentially breaking changes ---------------------------- diff --git a/src/api/custom_syntax.rs b/src/api/custom_syntax.rs index f881ff88..91b89b2c 100644 --- a/src/api/custom_syntax.rs +++ b/src/api/custom_syntax.rs @@ -255,7 +255,8 @@ impl Engine { // Markers not in first position #[cfg(not(feature = "no_float"))] CUSTOM_SYNTAX_MARKER_FLOAT if !segments.is_empty() => s.into(), - // Standard or reserved keyword/symbol not in first position + + // Keyword/symbol not in first position _ if !segments.is_empty() && token.is_some() => { // Make it a custom keyword/symbol if it is disabled or reserved if (self @@ -274,6 +275,7 @@ impl Engine { } s.into() } + // Standard keyword in first position but not disabled _ if segments.is_empty() && token.as_ref().map_or(false, Token::is_standard_keyword) @@ -291,8 +293,11 @@ impl Engine { ) .into_err(Position::NONE)); } - // Identifier in first position - _ if segments.is_empty() && is_valid_identifier(s) => { + + // Identifier or symbol in first position + _ if segments.is_empty() + && (is_valid_identifier(s) || is_reserved_keyword_or_symbol(s)) => + { // Make it a custom keyword/symbol if it is disabled or reserved if self .disabled_symbols @@ -310,6 +315,7 @@ impl Engine { } s.into() } + // Anything else is an error _ => { return Err(LexError::ImproperSymbol( @@ -326,23 +332,20 @@ impl Engine { segments.push(seg); } - // If the syntax has no symbols, just ignore the registration + // If the syntax has nothing, just ignore the registration if segments.is_empty() { return Ok(self); } - // The first keyword is the discriminator + // The first keyword/symbol is the discriminator let key = segments[0].clone(); self.register_custom_syntax_with_state_raw( key, // Construct the parsing function - move |stream, _, _| { - if stream.len() >= segments.len() { - Ok(None) - } else { - Ok(Some(segments[stream.len()].clone())) - } + move |stream, _, _| match stream.len() { + len if len >= segments.len() => Ok(None), + len => Ok(Some(segments[len].clone())), }, scope_may_be_changed, move |context, expressions, _| func(context, expressions), diff --git a/src/parser.rs b/src/parser.rs index f4fd7ad5..a45ce89d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3792,7 +3792,7 @@ impl Engine { (.., pos) => { return Err(PERR::MissingToken( Token::Pipe.into(), - "to close the parameters list of anonymous function".into(), + "to close the parameters list of anonymous function or closure".into(), ) .into_err(pos)) } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 4ec35bea..40d49057 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2066,7 +2066,7 @@ pub const fn is_id_continue(x: char) -> bool { x.is_ascii_alphanumeric() || x == '_' } -/// Is a piece of syntax a reserved keyword or symbol? +/// Is a piece of syntax a reserved keyword or reserved symbol? #[must_use] pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { match syntax { diff --git a/tests/custom_syntax.rs b/tests/custom_syntax.rs index 32382acd..ec71e904 100644 --- a/tests/custom_syntax.rs +++ b/tests/custom_syntax.rs @@ -251,6 +251,62 @@ fn test_custom_syntax() -> Result<(), Box> { Ok(()) } +#[test] +fn test_custom_syntax_matrix() -> Result<(), Box> { + let mut engine = Engine::new(); + + engine.disable_symbol("|"); + + engine.register_custom_syntax( + [ + "@", // + "|", "$expr$", "$expr$", "$expr$", "|", // + "|", "$expr$", "$expr$", "$expr$", "|", // + "|", "$expr$", "$expr$", "$expr$", "|", + ], + false, + |context, inputs| { + let mut values = [[0; 3]; 3]; + + for y in 0..3 { + for x in 0..3 { + let offset = y * 3 + x; + + match context.eval_expression_tree(&inputs[offset])?.as_int() { + Ok(v) => values[y][x] = v, + Err(typ) => { + return Err(Box::new(EvalAltResult::ErrorMismatchDataType( + "integer".to_string(), + typ.to_string(), + inputs[offset].position(), + ))) + } + } + } + } + + Ok(Dynamic::from(values)) + }, + )?; + + let r = engine.eval::<[[INT; 3]; 3]>( + " + let a = 42; + let b = 123; + let c = 1; + let d = 99; + + @| a b 0 | + | -b a 0 | + | 0 0 c*d | + ", + )?; + + assert_eq!(r, [[42, 123, 0], [-123, 42, 0], [0, 0, 99]]); + + Ok(()) +} + #[test] fn test_custom_syntax_raw() -> Result<(), Box> { let mut engine = Engine::new(); From 129a5c6e86e4a2997da87007ced539e33b00b15c Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 16:36:57 +0800 Subject: [PATCH 13/41] Use then/then_some to simplify. --- src/eval/stmt.rs | 8 ++------ src/func/call.rs | 43 +++++++++++++++++-------------------------- src/func/script.rs | 8 +++----- src/optimizer.rs | 6 +----- src/parser.rs | 23 +++++++---------------- src/tokenizer.rs | 6 +----- 6 files changed, 31 insertions(+), 63 deletions(-) diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 33a26958..4d58a9d7 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -139,12 +139,8 @@ impl Engine { // Built-in found auto_restore! { let orig_level = global.level; global.level += 1 } - let context = if need_context { - let source = global.source(); - Some((self, op_x_str, source, &*global, pos).into()) - } else { - None - }; + let context = need_context + .then_some((self, op_x_str, global.source(), &*global, pos).into()); return func(context, args).map(|_| ()); } } diff --git a/src/func/call.rs b/src/func/call.rs index 2abda94b..0165719c 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -398,11 +398,9 @@ impl Engine { let is_method = func.is_method(); let src = source.as_ref().map(|s| s.as_str()); - let context = if func.has_context() { - Some((self, name, src, &*global, pos).into()) - } else { - None - }; + let context = func + .has_context() + .then_some((self, name, src, &*global, pos).into()); let mut _result = if let Some(f) = func.get_plugin_fn() { if !f.is_pure() && !args.is_empty() && args[0].is_read_only() { @@ -1460,11 +1458,9 @@ impl Engine { Some(f) if f.is_plugin_fn() => { let f = f.get_plugin_fn().expect("plugin function"); - let context = if f.has_context() { - Some((self, fn_name, module.id(), &*global, pos).into()) - } else { - None - }; + let context = f + .has_context() + .then_some((self, fn_name, module.id(), &*global, pos).into()); if !f.is_pure() && !args.is_empty() && args[0].is_read_only() { Err(ERR::ErrorNonPureMethodCallOnConstant(fn_name.to_string(), pos).into()) } else { @@ -1475,18 +1471,16 @@ impl Engine { Some(f) if f.is_native() => { let func = f.get_native_fn().expect("native function"); - let context = if f.has_context() { - Some((self, fn_name, module.id(), &*global, pos).into()) - } else { - None - }; + let context = f + .has_context() + .then_some((self, fn_name, module.id(), &*global, pos).into()); func(context, args).and_then(|r| self.check_data_size(r, pos)) } Some(f) => unreachable!("unknown function type: {:?}", f), - None => { - let sig = if namespace.is_empty() { + None => Err(ERR::ErrorFunctionNotFound( + if namespace.is_empty() { self.gen_fn_call_signature(fn_name, args) } else { format!( @@ -1494,10 +1488,10 @@ impl Engine { crate::engine::NAMESPACE_SEPARATOR, self.gen_fn_call_signature(fn_name, args) ) - }; - - Err(ERR::ErrorFunctionNotFound(sig, pos).into()) - } + }, + pos, + ) + .into()), } } @@ -1605,11 +1599,8 @@ impl Engine { // Built-in found auto_restore! { let orig_level = global.level; global.level += 1 } - let context = if need_context { - Some((self, name.as_str(), None, &*global, pos).into()) - } else { - None - }; + let context = + need_context.then_some((self, name.as_str(), None, &*global, pos).into()); return func(context, operands); } diff --git a/src/func/script.rs b/src/func/script.rs index b864b4e8..18c09099 100644 --- a/src/func/script.rs +++ b/src/func/script.rs @@ -86,7 +86,7 @@ impl Engine { let orig_fn_resolution_caches_len = caches.fn_resolution_caches_len(); #[cfg(not(feature = "no_module"))] - let orig_constants = if let Some(environ) = _environ { + let orig_constants = _environ.map(|environ| { let EncapsulatedEnviron { lib, imports, @@ -100,10 +100,8 @@ impl Engine { global.lib.push(lib.clone()); - Some(mem::replace(&mut global.constants, constants.clone())) - } else { - None - }; + mem::replace(&mut global.constants, constants.clone()) + }); #[cfg(feature = "debugging")] if self.is_debugger_registered() { diff --git a/src/optimizer.rs b/src/optimizer.rs index a502a6f4..c702c0a0 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -1137,11 +1137,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) { _ if x.args.len() == 2 && x.op_token.is_some() && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => { if let Some(result) = get_builtin_binary_op_fn(x.op_token.as_ref().unwrap(), &arg_values[0], &arg_values[1]) .and_then(|(f, ctx)| { - let context = if ctx { - Some((state.engine, x.name.as_str(), None, &state.global, *pos).into()) - } else { - None - }; + let context = ctx.then_some((state.engine, x.name.as_str(), None, &state.global, *pos).into()); let (first, second) = arg_values.split_first_mut().unwrap(); (f)(context, &mut [ first, &mut second[0] ]).ok() }) { diff --git a/src/parser.rs b/src/parser.rs index a45ce89d..927d292b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -215,11 +215,7 @@ impl<'e, 's> ParseState<'e, 's> { self.allow_capture = true; } - let index = if hit_barrier { - None - } else { - NonZeroUsize::new(index) - }; + let index = (!hit_barrier).then(|| NonZeroUsize::new(index)).flatten(); (index, is_func_name) } @@ -2357,12 +2353,8 @@ impl Engine { let op = op_token.to_string(); let hash = calc_fn_hash(None, &op, 2); - let is_valid_script_function = is_valid_function_name(&op); - let operator_token = if is_valid_script_function { - None - } else { - Some(op_token.clone()) - }; + let native_only = !is_valid_function_name(&op); + let operator_token = native_only.then_some(op_token.clone()); let mut args = FnArgsVec::new_const(); args.push(root); @@ -2446,10 +2438,10 @@ impl Engine { .and_then(|m| m.get(s.as_str())) .map_or(false, Option::is_some) => { - op_base.hashes = if is_valid_script_function { - FnCallHashes::from_hash(calc_fn_hash(None, &s, 2)) - } else { + op_base.hashes = if native_only { FnCallHashes::from_native_only(calc_fn_hash(None, &s, 2)) + } else { + FnCallHashes::from_hash(calc_fn_hash(None, &s, 2)) }; op_base.into_fn_call_expr(pos) } @@ -3085,8 +3077,7 @@ impl Engine { let (id, id_pos) = parse_var_name(input)?; let (alias, alias_pos) = if match_token(input, Token::As).0 { - let (name, pos) = parse_var_name(input)?; - (Some(name), pos) + parse_var_name(input).map(|(name, pos)| (Some(name), pos))? } else { (None, Position::NONE) }; diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 40d49057..9a22fe55 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1229,11 +1229,7 @@ fn get_next_token_inner( // Still inside a comment? if state.comment_level > 0 { let start_pos = *pos; - let mut comment = if state.include_comments { - Some(String::new()) - } else { - None - }; + let mut comment = state.include_comments.then_some(String::new()); state.comment_level = scan_block_comment(stream, state.comment_level, pos, comment.as_mut()); From b371c524aee4bbef9e34d03b2b24d886ba779a96 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 17:14:14 +0800 Subject: [PATCH 14/41] Use indirection for doc. --- src/api/optimize.rs | 6 +++++- src/ast/ast.rs | 42 ++++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/api/optimize.rs b/src/api/optimize.rs index 92adec95..32927c0f 100644 --- a/src/api/optimize.rs +++ b/src/api/optimize.rs @@ -63,7 +63,11 @@ impl Engine { ); #[cfg(feature = "metadata")] - _new_ast.set_doc(std::mem::take(ast.doc_mut())); + if let Some(doc) = ast.doc_mut() { + _new_ast.set_doc(std::mem::take(doc)); + } else { + _new_ast.clear_doc(); + } _new_ast } diff --git a/src/ast/ast.rs b/src/ast/ast.rs index ea5e05e6..71f7cc22 100644 --- a/src/ast/ast.rs +++ b/src/ast/ast.rs @@ -23,7 +23,7 @@ pub struct AST { source: Option, /// [`AST`] documentation. #[cfg(feature = "metadata")] - doc: crate::SmartString, + doc: Option, /// Global statements. body: StmtBlock, /// Script-defined functions. @@ -98,7 +98,7 @@ impl AST { Self { source: None, #[cfg(feature = "metadata")] - doc: crate::SmartString::new_const(), + doc: None, body: StmtBlock::new(statements, Position::NONE, Position::NONE), #[cfg(not(feature = "no_function"))] lib: functions.into(), @@ -148,7 +148,7 @@ impl AST { Self { source: None, #[cfg(feature = "metadata")] - doc: crate::SmartString::new_const(), + doc: None, body: StmtBlock::NONE, #[cfg(not(feature = "no_function"))] lib: crate::Module::new().into(), @@ -202,14 +202,14 @@ impl AST { #[inline(always)] #[must_use] pub fn doc(&self) -> &str { - &self.doc + self.doc.as_ref().map(|s| s.as_str()).unwrap_or_default() } /// Clear the documentation. /// Exported under the `metadata` feature only. #[cfg(feature = "metadata")] #[inline(always)] pub fn clear_doc(&mut self) -> &mut Self { - self.doc.clear(); + self.doc = None; self } /// Get a mutable reference to the documentation. @@ -219,8 +219,8 @@ impl AST { #[inline(always)] #[must_use] #[allow(dead_code)] - pub(crate) fn doc_mut(&mut self) -> &mut crate::SmartString { - &mut self.doc + pub(crate) fn doc_mut(&mut self) -> Option<&mut crate::SmartString> { + self.doc.as_mut() } /// Set the documentation. /// @@ -228,7 +228,13 @@ impl AST { #[cfg(feature = "metadata")] #[inline(always)] pub(crate) fn set_doc(&mut self, doc: impl Into) { - self.doc = doc.into(); + let doc = doc.into(); + + if doc.is_empty() { + self.doc = None; + } else { + self.doc = Some(doc); + } } /// Get the statements. #[cfg(not(feature = "internals"))] @@ -598,11 +604,13 @@ impl AST { } #[cfg(feature = "metadata")] - if !other.doc.is_empty() { - if !_ast.doc.is_empty() { - _ast.doc.push('\n'); + if let Some(ref other_doc) = other.doc { + if let Some(ref mut ast_doc) = _ast.doc { + ast_doc.push('\n'); + ast_doc.push_str(other_doc); + } else { + _ast.doc = Some(other_doc.clone()); } - _ast.doc.push_str(other.doc()); } _ast @@ -698,11 +706,13 @@ impl AST { } #[cfg(feature = "metadata")] - if !other.doc.is_empty() { - if !self.doc.is_empty() { - self.doc.push('\n'); + if let Some(other_doc) = other.doc { + if let Some(ref mut self_doc) = self.doc { + self_doc.push('\n'); + self_doc.push_str(&other_doc); + } else { + self.doc = Some(other_doc); } - self.doc.push_str(&other.doc); } self From a8be9c4d64bbe94a949d7f0a0af0e498a4406e0d Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 17:52:11 +0800 Subject: [PATCH 15/41] Reduce size of AST. --- src/ast/ast.rs | 62 +++++++++++++++++++++++++++++++--------------- tests/optimizer.rs | 12 ++++----- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/ast/ast.rs b/src/ast/ast.rs index 71f7cc22..bff66b1d 100644 --- a/src/ast/ast.rs +++ b/src/ast/ast.rs @@ -23,9 +23,9 @@ pub struct AST { source: Option, /// [`AST`] documentation. #[cfg(feature = "metadata")] - doc: Option, + doc: Option>, /// Global statements. - body: StmtBlock, + body: Option>, /// Script-defined functions. #[cfg(not(feature = "no_function"))] lib: crate::SharedModule, @@ -54,7 +54,14 @@ impl fmt::Debug for AST { #[cfg(not(feature = "no_module"))] fp.field("resolver", &self.resolver); - fp.field("body", &self.body.as_slice()); + fp.field( + "body", + &self + .body + .as_deref() + .map(|b| b.as_slice()) + .unwrap_or_default(), + ); #[cfg(not(feature = "no_function"))] for (.., fn_def) in self.lib.iter_script_fn() { @@ -75,11 +82,13 @@ impl AST { statements: impl IntoIterator, #[cfg(not(feature = "no_function"))] functions: impl Into, ) -> Self { + let stmt = StmtBlock::new(statements, Position::NONE, Position::NONE); + Self { source: None, #[cfg(feature = "metadata")] doc: crate::SmartString::new_const(), - body: StmtBlock::new(statements, Position::NONE, Position::NONE), + body: (!stmt.is_empty()).then(|| stmt.into()), #[cfg(not(feature = "no_function"))] lib: functions.into(), #[cfg(not(feature = "no_module"))] @@ -95,11 +104,13 @@ impl AST { statements: impl IntoIterator, #[cfg(not(feature = "no_function"))] functions: impl Into, ) -> Self { + let stmt = StmtBlock::new(statements, Position::NONE, Position::NONE); + Self { source: None, #[cfg(feature = "metadata")] doc: None, - body: StmtBlock::new(statements, Position::NONE, Position::NONE), + body: (!stmt.is_empty()).then(|| stmt.into()), #[cfg(not(feature = "no_function"))] lib: functions.into(), #[cfg(not(feature = "no_module"))] @@ -149,7 +160,7 @@ impl AST { source: None, #[cfg(feature = "metadata")] doc: None, - body: StmtBlock::NONE, + body: None, #[cfg(not(feature = "no_function"))] lib: crate::Module::new().into(), #[cfg(not(feature = "no_module"))] @@ -220,7 +231,7 @@ impl AST { #[must_use] #[allow(dead_code)] pub(crate) fn doc_mut(&mut self) -> Option<&mut crate::SmartString> { - self.doc.as_mut() + self.doc.as_deref_mut() } /// Set the documentation. /// @@ -233,7 +244,7 @@ impl AST { if doc.is_empty() { self.doc = None; } else { - self.doc = Some(doc); + self.doc = Some(doc.into()); } } /// Get the statements. @@ -249,14 +260,20 @@ impl AST { #[inline(always)] #[must_use] pub fn statements(&self) -> &[Stmt] { - self.body.statements() + self.body + .as_deref() + .map(StmtBlock::statements) + .unwrap_or_default() } /// Extract the statements. #[allow(dead_code)] #[inline(always)] #[must_use] pub(crate) fn take_statements(&mut self) -> StmtBlockContainer { - self.body.take_statements() + self.body + .as_deref_mut() + .map(StmtBlock::take_statements) + .unwrap_or_default() } /// Does this [`AST`] contain script-defined functions? /// @@ -350,7 +367,7 @@ impl AST { source: self.source.clone(), #[cfg(feature = "metadata")] doc: self.doc.clone(), - body: StmtBlock::NONE, + body: None, lib: lib.into(), #[cfg(not(feature = "no_module"))] resolver: self.resolver.clone(), @@ -548,15 +565,15 @@ impl AST { other: &Self, _filter: impl Fn(FnNamespace, FnAccess, bool, &str, usize) -> bool, ) -> Self { - let merged = match (self.body.is_empty(), other.body.is_empty()) { - (false, false) => { - let mut body = self.body.clone(); - body.extend(other.body.iter().cloned()); + let merged = match (&self.body, &other.body) { + (Some(body), Some(other)) => { + let mut body = body.as_ref().clone(); + body.extend(other.iter().cloned()); body } - (false, true) => self.body.clone(), - (true, false) => other.body.clone(), - (true, true) => StmtBlock::NONE, + (Some(body), None) => body.as_ref().clone(), + (None, Some(body)) => body.as_ref().clone(), + (None, None) => StmtBlock::NONE, }; #[cfg(not(feature = "no_function"))] @@ -698,7 +715,12 @@ impl AST { } } - self.body.extend(other.body.into_iter()); + match (&mut self.body, other.body) { + (Some(body), Some(other)) => body.extend(other.into_iter()), + (Some(_), None) => (), + (None, body @ Some(_)) => self.body = body, + (None, None) => (), + } #[cfg(not(feature = "no_function"))] if !other.lib.is_empty() { @@ -795,7 +817,7 @@ impl AST { /// Clear all statements in the [`AST`], leaving only function definitions. #[inline(always)] pub fn clear_statements(&mut self) -> &mut Self { - self.body = StmtBlock::NONE; + self.body = None; self } /// Extract all top-level literal constant and/or variable definitions. diff --git a/tests/optimizer.rs b/tests/optimizer.rs index b72cab02..c52a3deb 100644 --- a/tests/optimizer.rs +++ b/tests/optimizer.rs @@ -89,21 +89,21 @@ fn test_optimizer_parse() -> Result<(), Box> { assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [Expr(123 @ 1:53)] }"# + r#"AST { source: None, doc: None, resolver: None, body: [Expr(123 @ 1:53)] }"# ); let ast = engine.compile("const DECISION = false; if DECISION { 42 } else { 123 }")?; assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [Var(("DECISION" @ 1:7, false @ 1:18, None), CONSTANT, 1:1), Expr(123 @ 1:51)] }"# + r#"AST { source: None, doc: None, resolver: None, body: [Var(("DECISION" @ 1:7, false @ 1:18, None), CONSTANT, 1:1), Expr(123 @ 1:51)] }"# ); let ast = engine.compile("if 1 == 2 { 42 }")?; assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [] }"# + r#"AST { source: None, doc: None, resolver: None, body: [] }"# ); engine.set_optimization_level(OptimizationLevel::Full); @@ -112,14 +112,14 @@ fn test_optimizer_parse() -> Result<(), Box> { assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [Expr(42 @ 1:1)] }"# + r#"AST { source: None, doc: None, resolver: None, body: [Expr(42 @ 1:1)] }"# ); let ast = engine.compile("NUMBER")?; assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [Expr(Variable(NUMBER) @ 1:1)] }"# + r#"AST { source: None, doc: None, resolver: None, body: [Expr(Variable(NUMBER) @ 1:1)] }"# ); let mut module = Module::new(); @@ -131,7 +131,7 @@ fn test_optimizer_parse() -> Result<(), Box> { assert_eq!( format!("{ast:?}"), - r#"AST { source: None, doc: "", resolver: None, body: [Expr(42 @ 1:1)] }"# + r#"AST { source: None, doc: None, resolver: None, body: [Expr(42 @ 1:1)] }"# ); Ok(()) From c58b52f2cb42105ef2a7d25e313e9cdcdee938e8 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 18:03:21 +0800 Subject: [PATCH 16/41] Fix builds. --- src/ast/ast.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ast/ast.rs b/src/ast/ast.rs index bff66b1d..8a23d96e 100644 --- a/src/ast/ast.rs +++ b/src/ast/ast.rs @@ -252,7 +252,10 @@ impl AST { #[inline(always)] #[must_use] pub(crate) fn statements(&self) -> &[Stmt] { - self.body.statements() + self.body + .as_deref() + .map(StmtBlock::statements) + .unwrap_or_default() } /// _(internals)_ Get the statements. /// Exported under the `internals` feature only. From 14e205cabc50b45096ca260eaa7913466a60ea76 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 18:16:03 +0800 Subject: [PATCH 17/41] Satisfy msrv by removing then_some. --- src/eval/stmt.rs | 2 +- src/func/call.rs | 8 ++++---- src/optimizer.rs | 2 +- src/parser.rs | 2 +- src/tokenizer.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 4d58a9d7..99958724 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -140,7 +140,7 @@ impl Engine { auto_restore! { let orig_level = global.level; global.level += 1 } let context = need_context - .then_some((self, op_x_str, global.source(), &*global, pos).into()); + .then(|| (self, op_x_str, global.source(), &*global, pos).into()); return func(context, args).map(|_| ()); } } diff --git a/src/func/call.rs b/src/func/call.rs index 0165719c..490ce278 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -400,7 +400,7 @@ impl Engine { let context = func .has_context() - .then_some((self, name, src, &*global, pos).into()); + .then(|| (self, name, src, &*global, pos).into()); let mut _result = if let Some(f) = func.get_plugin_fn() { if !f.is_pure() && !args.is_empty() && args[0].is_read_only() { @@ -1460,7 +1460,7 @@ impl Engine { let f = f.get_plugin_fn().expect("plugin function"); let context = f .has_context() - .then_some((self, fn_name, module.id(), &*global, pos).into()); + .then(|| (self, fn_name, module.id(), &*global, pos).into()); if !f.is_pure() && !args.is_empty() && args[0].is_read_only() { Err(ERR::ErrorNonPureMethodCallOnConstant(fn_name.to_string(), pos).into()) } else { @@ -1473,7 +1473,7 @@ impl Engine { let func = f.get_native_fn().expect("native function"); let context = f .has_context() - .then_some((self, fn_name, module.id(), &*global, pos).into()); + .then(|| (self, fn_name, module.id(), &*global, pos).into()); func(context, args).and_then(|r| self.check_data_size(r, pos)) } @@ -1600,7 +1600,7 @@ impl Engine { auto_restore! { let orig_level = global.level; global.level += 1 } let context = - need_context.then_some((self, name.as_str(), None, &*global, pos).into()); + need_context.then(|| (self, name.as_str(), None, &*global, pos).into()); return func(context, operands); } diff --git a/src/optimizer.rs b/src/optimizer.rs index c702c0a0..c2dcb0c1 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -1137,7 +1137,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) { _ if x.args.len() == 2 && x.op_token.is_some() && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => { if let Some(result) = get_builtin_binary_op_fn(x.op_token.as_ref().unwrap(), &arg_values[0], &arg_values[1]) .and_then(|(f, ctx)| { - let context = ctx.then_some((state.engine, x.name.as_str(), None, &state.global, *pos).into()); + let context = ctx.then(|| (state.engine, x.name.as_str(), None, &state.global, *pos).into()); let (first, second) = arg_values.split_first_mut().unwrap(); (f)(context, &mut [ first, &mut second[0] ]).ok() }) { diff --git a/src/parser.rs b/src/parser.rs index 927d292b..6d4fd19b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2354,7 +2354,7 @@ impl Engine { let op = op_token.to_string(); let hash = calc_fn_hash(None, &op, 2); let native_only = !is_valid_function_name(&op); - let operator_token = native_only.then_some(op_token.clone()); + let operator_token = native_only.then(|| op_token.clone()); let mut args = FnArgsVec::new_const(); args.push(root); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9a22fe55..2d27a948 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1229,7 +1229,7 @@ fn get_next_token_inner( // Still inside a comment? if state.comment_level > 0 { let start_pos = *pos; - let mut comment = state.include_comments.then_some(String::new()); + let mut comment = state.include_comments.then(|| String::new()); state.comment_level = scan_block_comment(stream, state.comment_level, pos, comment.as_mut()); From 5b200a6d3dbe63bb6cad153f3b9b7780530f0941 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 21 Feb 2023 18:20:25 +0800 Subject: [PATCH 18/41] Fix build. --- src/ast/ast.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/ast.rs b/src/ast/ast.rs index 8a23d96e..84057e6a 100644 --- a/src/ast/ast.rs +++ b/src/ast/ast.rs @@ -87,7 +87,7 @@ impl AST { Self { source: None, #[cfg(feature = "metadata")] - doc: crate::SmartString::new_const(), + doc: None, body: (!stmt.is_empty()).then(|| stmt.into()), #[cfg(not(feature = "no_function"))] lib: functions.into(), From c2a8c342bbeb35984ddb5440c8bd5bdd522f2c06 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Fri, 24 Feb 2023 14:24:10 +0800 Subject: [PATCH 19/41] Modify hashing function. --- src/func/hashing.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/func/hashing.rs b/src/func/hashing.rs index 58c4202b..acc94fae 100644 --- a/src/func/hashing.rs +++ b/src/func/hashing.rs @@ -86,7 +86,7 @@ pub fn calc_var_hash<'a>(namespace: impl IntoIterator, var_name: } count += 1; }); - count.hash(s); + s.write_usize(count); var_name.hash(s); s.finish() @@ -120,9 +120,9 @@ pub fn calc_fn_hash<'a>( } count += 1; }); - count.hash(s); + s.write_usize(count); fn_name.hash(s); - num.hash(s); + s.write_usize(num); s.finish() } @@ -134,13 +134,12 @@ pub fn calc_fn_hash<'a>( #[must_use] pub fn calc_fn_hash_full(base: u64, params: impl IntoIterator) -> u64 { let s = &mut get_hasher(); - base.hash(s); let mut count = 0; params.into_iter().for_each(|t| { t.hash(s); count += 1; }); - count.hash(s); + s.write_usize(count); - s.finish() + s.finish() ^ base } From 51581cdef53ca950efc38790a1a1eeef4db57d19 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 25 Feb 2023 19:57:19 +0800 Subject: [PATCH 20/41] Make Engine faster to create. --- src/api/compile.rs | 29 +++++++-- src/api/eval.rs | 11 +++- src/api/events.rs | 4 +- src/api/json.rs | 12 +++- src/api/mod.rs | 7 ++- src/api/options.rs | 38 ++++++------ src/api/register.rs | 26 ++++---- src/api/run.rs | 13 +++- src/engine.rs | 125 +++++++++++++++++++------------------- src/eval/stmt.rs | 2 +- src/func/call.rs | 28 ++++++--- src/packages/lang_core.rs | 1 + src/parser.rs | 3 +- 13 files changed, 184 insertions(+), 115 deletions(-) diff --git a/src/api/compile.rs b/src/api/compile.rs index e222e403..6d506b41 100644 --- a/src/api/compile.rs +++ b/src/api/compile.rs @@ -2,6 +2,7 @@ use crate::func::native::locked_write; use crate::parser::{ParseResult, ParseState}; +use crate::types::StringsInterner; use crate::{Engine, OptimizationLevel, Scope, AST}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -126,7 +127,7 @@ impl Engine { let path = path.clone(); match self - .module_resolver + .module_resolver() .resolve_ast(self, None, &path, crate::Position::NONE) { Some(Ok(module_ast)) => collect_imports(&module_ast, &resolver, &mut imports), @@ -135,7 +136,7 @@ impl Engine { } let module = - self.module_resolver + self.module_resolver() .resolve(self, None, &path, crate::Position::NONE)?; let module = shared_take_or_clone(module); @@ -223,7 +224,17 @@ impl Engine { optimization_level: OptimizationLevel, ) -> ParseResult { let (stream, tc) = self.lex_raw(scripts.as_ref(), self.token_mapper.as_deref()); - let interned_strings = &mut *locked_write(&self.interned_strings); + + let mut interner; + let mut guard; + let interned_strings = if let Some(ref interner) = self.interned_strings { + guard = locked_write(interner); + &mut *guard + } else { + interner = StringsInterner::new(); + &mut interner + }; + let state = &mut ParseState::new(scope, interned_strings, tc); let mut _ast = self.parse(stream.peekable(), state, optimization_level)?; #[cfg(feature = "metadata")] @@ -294,7 +305,17 @@ impl Engine { ) -> ParseResult { let scripts = [script]; let (stream, t) = self.lex_raw(&scripts, self.token_mapper.as_deref()); - let interned_strings = &mut *locked_write(&self.interned_strings); + + let mut interner; + let mut guard; + let interned_strings = if let Some(ref interner) = self.interned_strings { + guard = locked_write(interner); + &mut *guard + } else { + interner = StringsInterner::new(); + &mut interner + }; + let state = &mut ParseState::new(Some(scope), interned_strings, t); self.parse_global_expr(stream.peekable(), state, |_| {}, self.optimization_level) } diff --git a/src/api/eval.rs b/src/api/eval.rs index 7c00e01e..3882fae5 100644 --- a/src/api/eval.rs +++ b/src/api/eval.rs @@ -4,6 +4,7 @@ use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::native::locked_write; use crate::parser::ParseState; use crate::types::dynamic::Variant; +use crate::types::StringsInterner; use crate::{ Dynamic, Engine, OptimizationLevel, Position, RhaiResult, RhaiResultOf, Scope, AST, ERR, }; @@ -119,7 +120,15 @@ impl Engine { ) -> RhaiResultOf { let scripts = [script]; let ast = { - let interned_strings = &mut *locked_write(&self.interned_strings); + let mut interner; + let mut guard; + let interned_strings = if let Some(ref interner) = self.interned_strings { + guard = locked_write(interner); + &mut *guard + } else { + interner = StringsInterner::new(); + &mut interner + }; let (stream, tc) = self.lex_raw(&scripts, self.token_mapper.as_deref()); diff --git a/src/api/events.rs b/src/api/events.rs index 09b9ce64..1b7d0749 100644 --- a/src/api/events.rs +++ b/src/api/events.rs @@ -286,7 +286,7 @@ impl Engine { /// ``` #[inline(always)] pub fn on_print(&mut self, callback: impl Fn(&str) + SendSync + 'static) -> &mut Self { - self.print = Box::new(callback); + self.print = Some(Box::new(callback)); self } /// Override default action of `debug` (print to stdout using [`println!`]) @@ -336,7 +336,7 @@ impl Engine { &mut self, callback: impl Fn(&str, Option<&str>, Position) + SendSync + 'static, ) -> &mut Self { - self.debug = Box::new(callback); + self.debug = Some(Box::new(callback)); self } /// _(debugging)_ Register a callback for debugging. diff --git a/src/api/json.rs b/src/api/json.rs index 24ad805d..ee266d24 100644 --- a/src/api/json.rs +++ b/src/api/json.rs @@ -4,6 +4,7 @@ use crate::func::native::locked_write; use crate::parser::{ParseSettingFlags, ParseState}; use crate::tokenizer::Token; +use crate::types::StringsInterner; use crate::{Engine, LexError, Map, OptimizationLevel, RhaiResultOf}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -115,7 +116,16 @@ impl Engine { ); let ast = { - let interned_strings = &mut *locked_write(&self.interned_strings); + let mut interner; + let mut guard; + let interned_strings = if let Some(ref interner) = self.interned_strings { + guard = locked_write(interner); + &mut *guard + } else { + interner = StringsInterner::new(); + &mut interner + }; + let state = &mut ParseState::new(None, interned_strings, tokenizer_control); self.parse_global_expr( diff --git a/src/api/mod.rs b/src/api/mod.rs index e9cc9c8a..50fbdaae 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -54,7 +54,10 @@ impl Engine { #[inline(always)] #[must_use] pub fn module_resolver(&self) -> &dyn crate::ModuleResolver { - &*self.module_resolver + const DUMMY_RESOLVER: crate::module::resolvers::DummyModuleResolver = + crate::module::resolvers::DummyModuleResolver; + + self.module_resolver.as_deref().unwrap_or(&DUMMY_RESOLVER) } /// Set the module resolution service used by the [`Engine`]. @@ -66,7 +69,7 @@ impl Engine { &mut self, resolver: impl crate::ModuleResolver + 'static, ) -> &mut Self { - self.module_resolver = Box::new(resolver); + self.module_resolver = Some(Box::new(resolver)); self } diff --git a/src/api/options.rs b/src/api/options.rs index 057a9ac3..4c23f4c9 100644 --- a/src/api/options.rs +++ b/src/api/options.rs @@ -38,24 +38,26 @@ impl LangOptions { /// Create a new [`LangOptions`] with default values. #[inline(always)] #[must_use] - pub fn new() -> Self { - Self::IF_EXPR - | Self::SWITCH_EXPR - | Self::LOOP_EXPR - | Self::STMT_EXPR - | Self::LOOPING - | Self::SHADOWING - | Self::FAST_OPS - | { - #[cfg(not(feature = "no_function"))] - { - Self::ANON_FN - } - #[cfg(feature = "no_function")] - { - Self::empty() - } - } + pub const fn new() -> Self { + Self::from_bits_truncate( + Self::IF_EXPR.bits() + | Self::SWITCH_EXPR.bits() + | Self::LOOP_EXPR.bits() + | Self::STMT_EXPR.bits() + | Self::LOOPING.bits() + | Self::SHADOWING.bits() + | Self::FAST_OPS.bits() + | { + #[cfg(not(feature = "no_function"))] + { + Self::ANON_FN.bits() + } + #[cfg(feature = "no_function")] + { + Self::empty().bits() + } + }, + ) } } diff --git a/src/api/register.rs b/src/api/register.rs index a2d26944..43c43a72 100644 --- a/src/api/register.rs +++ b/src/api/register.rs @@ -1,6 +1,7 @@ //! Module that defines the public function/module registration API of [`Engine`]. use crate::func::{FnCallArgs, RegisterNativeFunction, SendSync}; +use crate::module::ModuleFlags; use crate::types::dynamic::Variant; use crate::{ Engine, FnAccess, FnNamespace, Identifier, Module, NativeCallContext, RhaiResultOf, Shared, @@ -14,20 +15,18 @@ use std::prelude::v1::*; use crate::func::register::Mut; impl Engine { - /// Get the global namespace module (which is the fist module in `global_modules`). - #[inline(always)] - #[allow(dead_code)] - #[must_use] - pub(crate) fn global_namespace(&self) -> &Module { - self.global_modules.first().unwrap() - } /// Get a mutable reference to the global namespace module /// (which is the first module in `global_modules`). #[inline(always)] #[must_use] - pub(crate) fn global_namespace_mut(&mut self) -> &mut Module { - let module = self.global_modules.first_mut().unwrap(); - Shared::get_mut(module).expect("not shared") + fn global_namespace_mut(&mut self) -> &mut Module { + if self.global_modules.is_empty() { + let mut global_namespace = Module::new(); + global_namespace.flags |= ModuleFlags::INTERNAL; + self.global_modules.push(global_namespace.into()); + } + + Shared::get_mut(self.global_modules.first_mut().unwrap()).expect("not shared") } /// Register a custom function with the [`Engine`]. /// @@ -677,6 +676,9 @@ impl Engine { /// modules are searched in reverse order. #[inline(always)] pub fn register_global_module(&mut self, module: SharedModule) -> &mut Self { + // Make sure the global namespace is created. + let _ = self.global_namespace_mut(); + // Insert the module into the front. // The first module is always the global namespace. self.global_modules.insert(1, module); @@ -779,7 +781,9 @@ impl Engine { pub fn gen_fn_signatures(&self, include_packages: bool) -> Vec { let mut signatures = Vec::with_capacity(64); - signatures.extend(self.global_namespace().gen_fn_signatures()); + if let Some(global_namespace) = self.global_modules.first() { + signatures.extend(global_namespace.gen_fn_signatures()); + } #[cfg(not(feature = "no_module"))] for (name, m) in self.global_sub_modules.as_deref().into_iter().flatten() { diff --git a/src/api/run.rs b/src/api/run.rs index c6e14f13..1fe1e476 100644 --- a/src/api/run.rs +++ b/src/api/run.rs @@ -3,6 +3,7 @@ use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::native::locked_write; use crate::parser::ParseState; +use crate::types::StringsInterner; use crate::{Engine, RhaiResultOf, Scope, AST}; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -59,7 +60,17 @@ impl Engine { let scripts = [script]; let ast = { let (stream, tc) = self.lex_raw(&scripts, self.token_mapper.as_deref()); - let interned_strings = &mut *locked_write(&self.interned_strings); + + let mut interner; + let mut guard; + let interned_strings = if let Some(ref interner) = self.interned_strings { + guard = locked_write(interner); + &mut *guard + } else { + interner = StringsInterner::new(); + &mut interner + }; + let state = &mut ParseState::new(Some(scope), interned_strings, tc); self.parse(stream.peekable(), state, self.optimization_level)? }; diff --git a/src/engine.rs b/src/engine.rs index 750b35f0..3175c611 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -5,13 +5,11 @@ use crate::func::native::{ locked_write, OnDebugCallback, OnDefVarCallback, OnParseTokenCallback, OnPrintCallback, OnVarCallback, }; -use crate::module::ModuleFlags; use crate::packages::{Package, StandardPackage}; use crate::tokenizer::Token; use crate::types::StringsInterner; use crate::{ - Dynamic, Identifier, ImmutableString, Locked, Module, OptimizationLevel, SharedModule, - StaticVec, + Dynamic, Identifier, ImmutableString, Locked, OptimizationLevel, SharedModule, StaticVec, }; #[cfg(feature = "no_std")] use std::prelude::v1::*; @@ -103,10 +101,10 @@ pub struct Engine { /// A module resolution service. #[cfg(not(feature = "no_module"))] - pub(crate) module_resolver: Box, + pub(crate) module_resolver: Option>, /// Strings interner. - pub(crate) interned_strings: Locked, + pub(crate) interned_strings: Option>>, /// A set of symbols to disable. pub(crate) disabled_symbols: Option>>, @@ -127,9 +125,9 @@ pub struct Engine { pub(crate) token_mapper: Option>, /// Callback closure for implementing the `print` command. - pub(crate) print: Box, + pub(crate) print: Option>, /// Callback closure for implementing the `debug` command. - pub(crate) debug: Box, + pub(crate) debug: Option>, /// Callback closure for progress reporting. #[cfg(not(feature = "unchecked"))] pub(crate) progress: Option>, @@ -231,6 +229,49 @@ pub fn make_setter(id: &str) -> Identifier { } impl Engine { + /// An empty [`Engine`]. + pub const EMPTY: Self = Self { + global_modules: StaticVec::new_const(), + + #[cfg(not(feature = "no_module"))] + global_sub_modules: None, + + #[cfg(not(feature = "no_module"))] + module_resolver: None, + + interned_strings: None, + disabled_symbols: None, + #[cfg(not(feature = "no_custom_syntax"))] + custom_keywords: None, + #[cfg(not(feature = "no_custom_syntax"))] + custom_syntax: None, + + def_var_filter: None, + resolve_var: None, + token_mapper: None, + + print: None, + debug: None, + + #[cfg(not(feature = "unchecked"))] + progress: None, + + options: LangOptions::new(), + + def_tag: Dynamic::UNIT, + + #[cfg(not(feature = "no_optimize"))] + optimization_level: OptimizationLevel::Simple, + #[cfg(feature = "no_optimize")] + optimization_level: (), + + #[cfg(not(feature = "unchecked"))] + limits: crate::api::limits::Limits::new(), + + #[cfg(feature = "debugging")] + debugger_interface: None, + }; + /// Create a new [`Engine`]. #[inline] #[must_use] @@ -242,22 +283,25 @@ impl Engine { #[cfg(not(feature = "no_std"))] #[cfg(not(target_family = "wasm"))] { - engine.module_resolver = Box::new(crate::module::resolvers::FileModuleResolver::new()); + engine.module_resolver = + Some(Box::new(crate::module::resolvers::FileModuleResolver::new())); } + engine.interned_strings = Some(Locked::new(StringsInterner::new()).into()); + // default print/debug implementations #[cfg(not(feature = "no_std"))] #[cfg(not(target_family = "wasm"))] { - engine.print = Box::new(|s| println!("{s}")); - engine.debug = Box::new(|s, source, pos| match (source, pos) { + engine.print = Some(Box::new(|s| println!("{s}"))); + engine.debug = Some(Box::new(|s, source, pos| match (source, pos) { (Some(source), crate::Position::NONE) => println!("{source} | {s}"), #[cfg(not(feature = "no_position"))] (Some(source), pos) => println!("{source} @ {pos:?} | {s}"), (None, crate::Position::NONE) => println!("{s}"), #[cfg(not(feature = "no_position"))] (None, pos) => println!("{pos:?} | {s}"), - }); + })); } engine.register_global_module(StandardPackage::new().as_shared_module()); @@ -270,55 +314,8 @@ impl Engine { /// Use [`register_global_module`][Engine::register_global_module] to add packages of functions. #[inline] #[must_use] - pub fn new_raw() -> Self { - let mut engine = Self { - global_modules: StaticVec::new_const(), - - #[cfg(not(feature = "no_module"))] - global_sub_modules: None, - - #[cfg(not(feature = "no_module"))] - module_resolver: Box::new(crate::module::resolvers::DummyModuleResolver::new()), - - interned_strings: StringsInterner::new().into(), - disabled_symbols: None, - #[cfg(not(feature = "no_custom_syntax"))] - custom_keywords: None, - #[cfg(not(feature = "no_custom_syntax"))] - custom_syntax: None, - - def_var_filter: None, - resolve_var: None, - token_mapper: None, - - print: Box::new(|_| {}), - debug: Box::new(|_, _, _| {}), - - #[cfg(not(feature = "unchecked"))] - progress: None, - - options: LangOptions::new(), - - def_tag: Dynamic::UNIT, - - #[cfg(not(feature = "no_optimize"))] - optimization_level: OptimizationLevel::Simple, - #[cfg(feature = "no_optimize")] - optimization_level: (), - - #[cfg(not(feature = "unchecked"))] - limits: crate::api::limits::Limits::new(), - - #[cfg(feature = "debugging")] - debugger_interface: None, - }; - - // Add the global namespace module - let mut global_namespace = Module::new(); - global_namespace.flags |= ModuleFlags::INTERNAL; - engine.global_modules.push(global_namespace.into()); - - engine + pub const fn new_raw() -> Self { + Self::EMPTY } /// Get an interned [string][ImmutableString]. @@ -338,13 +335,17 @@ impl Engine { /// [`Engine`] keeps a cache of [`ImmutableString`] instances and tries to avoid new allocations /// when an existing instance is found. #[cfg(feature = "internals")] - #[inline(always)] + #[inline] #[must_use] pub fn get_interned_string( &self, string: impl AsRef + Into, ) -> ImmutableString { - locked_write(&self.interned_strings).get(string) + if let Some(ref interner) = self.interned_strings { + locked_write(interner).get(string) + } else { + string.into() + } } /// Get an empty [`ImmutableString`] which refers to a shared instance. diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 99958724..0ed2bf37 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -806,7 +806,7 @@ impl Engine { ) .or_else(|| { Some( - self.module_resolver + self.module_resolver() .resolve_raw(self, global, scope, &path, path_pos), ) }) diff --git a/src/func/call.rs b/src/func/call.rs index 490ce278..131089fa 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -459,18 +459,26 @@ impl Engine { // See if the function match print/debug (which requires special processing) return Ok(match name { KEYWORD_PRINT => { - let text = result.into_immutable_string().map_err(|typ| { - let t = self.map_type_name(type_name::()).into(); - ERR::ErrorMismatchOutputType(t, typ.into(), pos) - })?; - ((*self.print)(&text).into(), false) + if let Some(ref print) = self.print { + let text = result.into_immutable_string().map_err(|typ| { + let t = self.map_type_name(type_name::()).into(); + ERR::ErrorMismatchOutputType(t, typ.into(), pos) + })?; + ((print)(&text).into(), false) + } else { + (Dynamic::UNIT, false) + } } KEYWORD_DEBUG => { - let text = result.into_immutable_string().map_err(|typ| { - let t = self.map_type_name(type_name::()).into(); - ERR::ErrorMismatchOutputType(t, typ.into(), pos) - })?; - ((*self.debug)(&text, global.source(), pos).into(), false) + if let Some(ref debug) = self.debug { + let text = result.into_immutable_string().map_err(|typ| { + let t = self.map_type_name(type_name::()).into(); + ERR::ErrorMismatchOutputType(t, typ.into(), pos) + })?; + ((debug)(&text, global.source(), pos).into(), false) + } else { + (Dynamic::UNIT, false) + } } _ => (result, is_method), }); diff --git a/src/packages/lang_core.rs b/src/packages/lang_core.rs index 229113d3..1e5a736b 100644 --- a/src/packages/lang_core.rs +++ b/src/packages/lang_core.rs @@ -7,6 +7,7 @@ use crate::{Dynamic, RhaiResultOf, ERR, INT}; use std::prelude::v1::*; #[cfg(not(feature = "no_float"))] +#[cfg(not(feature = "no_std"))] use crate::FLOAT; def_package! { diff --git a/src/parser.rs b/src/parser.rs index 6d4fd19b..300d44f0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2354,7 +2354,6 @@ impl Engine { let op = op_token.to_string(); let hash = calc_fn_hash(None, &op, 2); let native_only = !is_valid_function_name(&op); - let operator_token = native_only.then(|| op_token.clone()); let mut args = FnArgsVec::new_const(); args.push(root); @@ -2366,7 +2365,7 @@ impl Engine { name: state.get_interned_string(&op), hashes: FnCallHashes::from_native_only(hash), args, - op_token: operator_token, + op_token: native_only.then(|| op_token.clone()), capture_parent_scope: false, }; From 5f09b5d9d9a28a9eb367ece4fef77fef51cc6664 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 25 Feb 2023 20:58:40 +0800 Subject: [PATCH 21/41] Fix builds. --- CHANGELOG.md | 1 + src/engine.rs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6be94fc5..07b73d9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Potentially breaking changes Enhancements ------------ +* `Engine::new_raw` is now `const` and runs very fast, delaying all other initialization until first use. * The functions `min` and `max` are added for numbers. * Range cases in `switch` statements now also match floating-point and decimal values. In order to support this, however, small numeric ranges cases are no longer unrolled. * Loading a module via `import` now gives the module access to the current scope, including variables and constants defined inside. diff --git a/src/engine.rs b/src/engine.rs index 3175c611..fd4e31ef 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -326,7 +326,11 @@ impl Engine { &self, string: impl AsRef + Into, ) -> ImmutableString { - locked_write(&self.interned_strings).get(string) + if let Some(ref interner) = self.interned_strings { + locked_write(interner).get(string) + } else { + string.into() + } } /// _(internals)_ Get an interned [string][ImmutableString]. From 9a5221f60d8b3f5191178e11db01f500be15cdb3 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 25 Feb 2023 21:41:11 +0800 Subject: [PATCH 22/41] Add Engine::RAW. --- src/engine.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index fd4e31ef..255ab7eb 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -229,8 +229,8 @@ pub fn make_setter(id: &str) -> Identifier { } impl Engine { - /// An empty [`Engine`]. - pub const EMPTY: Self = Self { + /// An empty raw [`Engine`]. + pub const RAW: Self = Self { global_modules: StaticVec::new_const(), #[cfg(not(feature = "no_module"))] @@ -310,12 +310,15 @@ impl Engine { } /// Create a new [`Engine`] with minimal built-in functions. + /// It returns a copy of [`Engine::RAW`]. + /// + /// This is useful for creating a custom scripting engine with only the functions you need. /// /// Use [`register_global_module`][Engine::register_global_module] to add packages of functions. #[inline] #[must_use] pub const fn new_raw() -> Self { - Self::EMPTY + Self::RAW } /// Get an interned [string][ImmutableString]. From 86ccb370297e1efd0f1645d634e3c5a9fc1bc7db Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 28 Feb 2023 22:33:44 +0800 Subject: [PATCH 23/41] Simplify code. --- src/ast/ast.rs | 13 ++----------- src/ast/expr.rs | 6 +----- src/eval/cache.rs | 2 +- src/eval/chaining.rs | 4 +--- src/eval/stmt.rs | 22 +++++++++++----------- src/func/call.rs | 11 ++++------- src/module/mod.rs | 26 ++++---------------------- src/module/resolvers/file.rs | 9 +-------- src/module/resolvers/stat.rs | 10 ++-------- src/parser.rs | 3 +-- 10 files changed, 28 insertions(+), 78 deletions(-) diff --git a/src/ast/ast.rs b/src/ast/ast.rs index 84057e6a..139f5634 100644 --- a/src/ast/ast.rs +++ b/src/ast/ast.rs @@ -189,11 +189,7 @@ impl AST { .as_mut() .map(|m| m.set_id(source.clone())); - if source.is_empty() { - self.source = None; - } else { - self.source = Some(source); - } + self.source = (!source.is_empty()).then(|| source); self } @@ -240,12 +236,7 @@ impl AST { #[inline(always)] pub(crate) fn set_doc(&mut self, doc: impl Into) { let doc = doc.into(); - - if doc.is_empty() { - self.doc = None; - } else { - self.doc = Some(doc.into()); - } + self.doc = (!doc.is_empty()).then(|| doc.into()); } /// Get the statements. #[cfg(not(feature = "internals"))] diff --git a/src/ast/expr.rs b/src/ast/expr.rs index 0152b72a..6add6903 100644 --- a/src/ast/expr.rs +++ b/src/ast/expr.rs @@ -248,11 +248,7 @@ impl FnCallExpr { #[inline] #[must_use] pub fn constant_args(&self) -> bool { - if self.args.is_empty() { - true - } else { - self.args.iter().all(Expr::is_constant) - } + self.args.is_empty() || self.args.iter().all(Expr::is_constant) } } diff --git a/src/eval/cache.rs b/src/eval/cache.rs index affe2b69..3e044574 100644 --- a/src/eval/cache.rs +++ b/src/eval/cache.rs @@ -63,8 +63,8 @@ impl Caches { #[inline] #[must_use] pub fn fn_resolution_cache_mut(&mut self) -> &mut FnResolutionCache { + // Push a new function resolution cache if the stack is empty if self.0.is_empty() { - // Push a new function resolution cache if the stack is empty self.push_fn_resolution_cache(); } self.0.last_mut().unwrap() diff --git a/src/eval/chaining.rs b/src/eval/chaining.rs index 09066c8c..22e8ae46 100644 --- a/src/eval/chaining.rs +++ b/src/eval/chaining.rs @@ -504,9 +504,7 @@ impl Engine { global, caches, scope, this_ptr, expr, rhs, idx_values, )?; - if !_arg_values.is_empty() { - idx_values.extend(_arg_values); - } + idx_values.extend(_arg_values); } #[cfg(not(feature = "no_object"))] diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 0ed2bf37..9a70177d 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -304,12 +304,14 @@ impl Engine { .as_bool() .map_err(|typ| self.make_type_mismatch_err::(typ, expr.position()))?; - if guard_val && !if_block.is_empty() { - self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) - } else if !guard_val && !else_block.is_empty() { - self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) - } else { - Ok(Dynamic::UNIT) + match guard_val { + true if !if_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) + } + false if !else_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) + } + _ => Ok(Dynamic::UNIT), } } @@ -512,12 +514,10 @@ impl Engine { auto_restore! { scope => rewind; let orig_scope_len = scope.len(); } // Add the loop variables - let counter_index = if counter.is_empty() { - usize::MAX - } else { + let counter_index = (!counter.is_empty()).then(|| { scope.push(counter.name.clone(), 0 as INT); scope.len() - 1 - }; + }); scope.push(var_name.name.clone(), ()); let index = scope.len() - 1; @@ -526,7 +526,7 @@ impl Engine { for (x, iter_value) in iter_func(iter_obj).enumerate() { // Increment counter - if counter_index < usize::MAX { + if let Some(counter_index) = counter_index { // As the variable increments from 0, this should always work // since any overflow will first be caught below. let index_value = x as INT; diff --git a/src/func/call.rs b/src/func/call.rs index 131089fa..a47c3495 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -816,7 +816,8 @@ impl Engine { if call_args.is_empty() { let typ = self.map_type_name(target.type_name()); return Err(self.make_type_mismatch_err::(typ, fn_call_pos)); - } else if !call_args[0].is_fnptr() { + } + if !call_args[0].is_fnptr() { let typ = self.map_type_name(call_args[0].type_name()); return Err(self.make_type_mismatch_err::(typ, first_arg_pos)); } @@ -1257,9 +1258,7 @@ impl Engine { } // Call with blank scope - if total_args == 0 && curry.is_empty() { - // No arguments - } else { + if total_args > 0 || !curry.is_empty() { // If the first argument is a variable, and there is no curried arguments, // convert to method-call style in order to leverage potential &mut first argument and // avoid cloning the value @@ -1330,9 +1329,7 @@ impl Engine { let args = &mut FnArgsVec::with_capacity(args_expr.len()); let mut first_arg_value = None; - if args_expr.is_empty() { - // No arguments - } else { + if !args_expr.is_empty() { // See if the first argument is a variable (not namespace-qualified). // If so, convert to method-call style in order to leverage potential &mut first argument // and avoid cloning the value diff --git a/src/module/mod.rs b/src/module/mod.rs index 0bed49c0..89ef6152 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -361,13 +361,7 @@ impl Module { #[inline(always)] pub fn set_id(&mut self, id: impl Into) -> &mut Self { let id = id.into(); - - if id.is_empty() { - self.id = None; - } else { - self.id = Some(id); - } - + self.id = (!id.is_empty()).then(|| id); self } @@ -2330,21 +2324,9 @@ impl Module { self.flags .set(ModuleFlags::INDEXED_GLOBAL_FUNCTIONS, has_global_functions); - self.all_variables = if variables.is_empty() { - None - } else { - Some(variables.into()) - }; - self.all_functions = if functions.is_empty() { - None - } else { - Some(functions.into()) - }; - self.all_type_iterators = if type_iterators.is_empty() { - None - } else { - Some(type_iterators.into()) - }; + self.all_variables = (!variables.is_empty()).then(|| variables.into()); + self.all_functions = (!functions.is_empty()).then(|| functions.into()); + self.all_type_iterators = (!type_iterators.is_empty()).then(|| type_iterators.into()); self.flags |= ModuleFlags::INDEXED; } diff --git a/src/module/resolvers/file.rs b/src/module/resolvers/file.rs index ea62c5fb..4ac9a47f 100644 --- a/src/module/resolvers/file.rs +++ b/src/module/resolvers/file.rs @@ -239,14 +239,7 @@ impl FileModuleResolver { if !self.cache_enabled { return false; } - - let cache = locked_read(&self.cache); - - if cache.is_empty() { - false - } else { - cache.contains_key(path.as_ref()) - } + locked_read(&self.cache).contains_key(path.as_ref()) } /// Empty the internal cache. #[inline] diff --git a/src/module/resolvers/stat.rs b/src/module/resolvers/stat.rs index bd2b2638..04449584 100644 --- a/src/module/resolvers/stat.rs +++ b/src/module/resolvers/stat.rs @@ -73,11 +73,7 @@ impl StaticModuleResolver { #[inline(always)] #[must_use] pub fn contains_path(&self, path: &str) -> bool { - if self.0.is_empty() { - false - } else { - self.0.contains_key(path) - } + self.0.contains_key(path) } /// Get an iterator of all the [modules][Module]. #[inline] @@ -123,9 +119,7 @@ impl StaticModuleResolver { /// Existing modules of the same path name are overwritten. #[inline] pub fn merge(&mut self, other: Self) -> &mut Self { - if !other.is_empty() { - self.0.extend(other.0.into_iter()); - } + self.0.extend(other.0.into_iter()); self } } diff --git a/src/parser.rs b/src/parser.rs index 300d44f0..182adc61 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -25,7 +25,6 @@ use bitflags::bitflags; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{ - collections::BTreeMap, convert::TryFrom, fmt, hash::{Hash, Hasher}, @@ -979,7 +978,7 @@ impl Engine { settings.pos = eat_token(input, Token::MapStart); let mut map = StaticVec::<(Ident, Expr)>::new(); - let mut template = BTreeMap::::new(); + let mut template = std::collections::BTreeMap::::new(); loop { const MISSING_RBRACE: &str = "to end this object map literal"; From a2f8d057f8436190612d18861f631f33855f63a0 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 2 Mar 2023 13:16:15 +0800 Subject: [PATCH 24/41] Add testing-environ feature. --- .github/workflows/build.yml | 44 ++++++++++++++++++------------------- Cargo.toml | 5 +++++ src/module/mod.rs | 30 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7dabfb37..74b0cd85 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,28 +43,28 @@ jobs: os: [ubuntu-latest] flags: - "" - - "--features debugging" - - "--features metadata,serde,internals" - - "--features unchecked,serde,metadata,internals,debugging" - - "--features sync,serde,metadata,internals,debugging" - - "--features no_position,serde,metadata,internals,debugging" - - "--features no_optimize,serde,metadata,internals,debugging" - - "--features no_float,serde,metadata,internals,debugging" - - "--features f32_float,serde,metadata,internals,debugging" - - "--features decimal,serde,metadata,internals,debugging" - - "--features no_custom_syntax,serde,metadata,internals,debugging" - - "--features no_float,decimal" - - "--tests --features only_i32,serde,metadata,internals,debugging" - - "--features only_i64,serde,metadata,internals,debugging" - - "--features no_index,serde,metadata,internals,debugging" - - "--features no_object,serde,metadata,internals,debugging" - - "--features no_function,serde,metadata,internals,debugging" - - "--features no_module,serde,metadata,internals,debugging" - - "--features no_time,serde,metadata,internals,debugging" - - "--features no_closure,serde,metadata,internals,debugging" - - "--features unicode-xid-ident,serde,metadata,internals,debugging" - - "--features sync,no_time,no_function,no_float,no_position,no_optimize,no_module,no_closure,no_custom_syntax,metadata,serde,unchecked,debugging" - - "--features no_time,no_function,no_float,no_position,no_index,no_object,no_optimize,no_module,no_closure,no_custom_syntax,unchecked" + - "--features testing-environ,debugging" + - "--features testing-environ,metadata,serde,internals" + - "--features testing-environ,unchecked,serde,metadata,internals,debugging" + - "--features testing-environ,sync,serde,metadata,internals,debugging" + - "--features testing-environ,no_position,serde,metadata,internals,debugging" + - "--features testing-environ,no_optimize,serde,metadata,internals,debugging" + - "--features testing-environ,no_float,serde,metadata,internals,debugging" + - "--features testing-environ,f32_float,serde,metadata,internals,debugging" + - "--features testing-environ,decimal,serde,metadata,internals,debugging" + - "--features testing-environ,no_custom_syntax,serde,metadata,internals,debugging" + - "--features testing-environ,no_float,decimal" + - "--tests --features testing-environ,only_i32,serde,metadata,internals,debugging" + - "--features testing-environ,only_i64,serde,metadata,internals,debugging" + - "--features testing-environ,no_index,serde,metadata,internals,debugging" + - "--features testing-environ,no_object,serde,metadata,internals,debugging" + - "--features testing-environ,no_function,serde,metadata,internals,debugging" + - "--features testing-environ,no_module,serde,metadata,internals,debugging" + - "--features testing-environ,no_time,serde,metadata,internals,debugging" + - "--features testing-environ,no_closure,serde,metadata,internals,debugging" + - "--features testing-environ,unicode-xid-ident,serde,metadata,internals,debugging" + - "--features testing-environ,sync,no_time,no_function,no_float,no_position,no_optimize,no_module,no_closure,no_custom_syntax,metadata,serde,unchecked,debugging" + - "--features testing-environ,no_time,no_function,no_float,no_position,no_index,no_object,no_optimize,no_module,no_closure,no_custom_syntax,unchecked" toolchain: [stable] experimental: [false] include: diff --git a/Cargo.toml b/Cargo.toml index 06c4ec6a..9730399c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,6 +117,11 @@ std = ["ahash/std", "num-traits/std", "smartstring/std"] ## Use [`stdweb`](https://crates.io/crates/stdweb) as JavaScript interface. stdweb = ["getrandom/js", "instant/stdweb"] +#! ### Features used in testing environments only + + ## Running under a testing environment. + testing-environ = [] + [[bin]] name = "rhai-repl" required-features = ["rustyline"] diff --git a/src/module/mod.rs b/src/module/mod.rs index 89ef6152..961b4ed6 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -715,8 +715,24 @@ impl Module { // None + function name + number of arguments. let num_params = fn_def.params.len(); let hash_script = crate::calc_fn_hash(None, &fn_def.name, num_params); + + #[cfg(feature = "testing-environ")] + if let Some(f) = self + .functions + .get_or_insert_with(|| { + StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) + }) + .get(&hash_script) + { + panic!( + "PANIC ATTACK!!! Function hash {} already exists when registering function {:#?}:\n{:#?}", + hash_script, fn_def, f + ); + } + #[cfg(feature = "metadata")] let params_info = fn_def.params.iter().map(Into::into).collect(); + self.functions .get_or_insert_with(|| { StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) @@ -1046,6 +1062,20 @@ impl Module { let hash_script = calc_fn_hash(None, name, param_types.len()); let hash_fn = calc_fn_hash_full(hash_script, param_types.iter().copied()); + #[cfg(feature = "testing-environ")] + if let Some(f) = self + .functions + .get_or_insert_with(|| { + StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) + }) + .get(&hash_script) + { + panic!( + "PANIC ATTACK!!! Function hash {} already exists when registering function {}:\n{:#?}", + hash_script, name, f + ); + } + if is_dynamic { self.dynamic_functions_filter .get_or_insert_with(Default::default) From 3e784d592da47846e9f3ce8f8860531edd54f439 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 2 Mar 2023 21:59:38 +0800 Subject: [PATCH 25/41] Check for more hash collisions. --- src/module/mod.rs | 101 +++++++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 33 deletions(-) diff --git a/src/module/mod.rs b/src/module/mod.rs index 961b4ed6..950a6d30 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -292,6 +292,11 @@ impl> AddAssign for Module { } } +#[inline(always)] +fn new_hash_map(size: usize) -> StraightHashMap { + StraightHashMap::with_capacity_and_hasher(size, Default::default()) +} + impl Module { /// Create a new [`Module`]. /// @@ -685,6 +690,16 @@ impl Module { if self.is_indexed() { let hash_var = crate::calc_var_hash(Some(""), &ident); + + // Catch hash collisions in testing environment only. + #[cfg(feature = "testing-environ")] + if let Some(_) = self.all_variables.as_ref().and_then(|f| f.get(&hash_var)) { + panic!( + "Hash {} already exists when registering variable {}", + hash_var, ident + ); + } + self.all_variables .get_or_insert_with(Default::default) .insert(hash_var, value.clone()); @@ -716,16 +731,11 @@ impl Module { let num_params = fn_def.params.len(); let hash_script = crate::calc_fn_hash(None, &fn_def.name, num_params); + // Catch hash collisions in testing environment only. #[cfg(feature = "testing-environ")] - if let Some(f) = self - .functions - .get_or_insert_with(|| { - StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) - }) - .get(&hash_script) - { + if let Some(f) = self.functions.as_ref().and_then(|f| f.get(&hash_script)) { panic!( - "PANIC ATTACK!!! Function hash {} already exists when registering function {:#?}:\n{:#?}", + "Hash {} already exists when registering function {:#?}:\n{:#?}", hash_script, fn_def, f ); } @@ -734,9 +744,7 @@ impl Module { let params_info = fn_def.params.iter().map(Into::into).collect(); self.functions - .get_or_insert_with(|| { - StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) - }) + .get_or_insert_with(|| new_hash_map(FN_MAP_SIZE)) .insert( hash_script, FuncInfo { @@ -1062,16 +1070,11 @@ impl Module { let hash_script = calc_fn_hash(None, name, param_types.len()); let hash_fn = calc_fn_hash_full(hash_script, param_types.iter().copied()); + // Catch hash collisions in testing environment only. #[cfg(feature = "testing-environ")] - if let Some(f) = self - .functions - .get_or_insert_with(|| { - StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) - }) - .get(&hash_script) - { + if let Some(f) = self.functions.as_ref().and_then(|f| f.get(&hash_script)) { panic!( - "PANIC ATTACK!!! Function hash {} already exists when registering function {}:\n{:#?}", + "Hash {} already exists when registering function {}:\n{:#?}", hash_script, name, f ); } @@ -1083,9 +1086,7 @@ impl Module { } self.functions - .get_or_insert_with(|| { - StraightHashMap::with_capacity_and_hasher(FN_MAP_SIZE, Default::default()) - }) + .get_or_insert_with(|| new_hash_map(FN_MAP_SIZE)) .insert( hash_fn, FuncInfo { @@ -1805,9 +1806,9 @@ impl Module { let others_len = functions.len(); for (&k, f) in functions.iter() { - let map = self.functions.get_or_insert_with(|| { - StraightHashMap::with_capacity_and_hasher(others_len, Default::default()) - }); + let map = self + .functions + .get_or_insert_with(|| new_hash_map(FN_MAP_SIZE)); map.reserve(others_len); map.entry(k).or_insert_with(|| f.clone()); } @@ -2283,6 +2284,16 @@ impl Module { if let Some(ref v) = module.variables { for (var_name, value) in v.iter() { let hash_var = crate::calc_var_hash(path.iter().copied(), var_name); + + // Catch hash collisions in testing environment only. + #[cfg(feature = "testing-environ")] + if let Some(_) = variables.get(&hash_var) { + panic!( + "Hash {} already exists when indexing variable {}", + hash_var, var_name + ); + } + variables.insert(hash_var, value.clone()); } } @@ -2298,6 +2309,15 @@ impl Module { for (&hash, f) in module.functions.iter().flatten() { match f.metadata.namespace { FnNamespace::Global => { + // Catch hash collisions in testing environment only. + #[cfg(feature = "testing-environ")] + if let Some(fx) = functions.get(&hash) { + panic!( + "Hash {} already exists when indexing function {:#?}:\n{:#?}", + hash, f.func, fx + ); + } + // Flatten all functions with global namespace functions.insert(hash, f.func.clone()); contains_indexed_global_functions = true; @@ -2315,6 +2335,16 @@ impl Module { f.metadata.name.as_str(), &f.metadata.param_types, ); + + // Catch hash collisions in testing environment only. + #[cfg(feature = "testing-environ")] + if let Some(fx) = functions.get(&hash_qualified_fn) { + panic!( + "Hash {} already exists when indexing function {:#?}:\n{:#?}", + hash_qualified_fn, f.func, fx + ); + } + functions.insert(hash_qualified_fn, f.func.clone()); } else if cfg!(not(feature = "no_function")) { let hash_qualified_script = crate::calc_fn_hash( @@ -2322,6 +2352,16 @@ impl Module { &f.metadata.name, f.metadata.num_params, ); + + // Catch hash collisions in testing environment only. + #[cfg(feature = "testing-environ")] + if let Some(fx) = functions.get(&hash_qualified_script) { + panic!( + "Hash {} already exists when indexing function {:#?}:\n{:#?}", + hash_qualified_script, f.func, fx + ); + } + functions.insert(hash_qualified_script, f.func.clone()); } } @@ -2331,14 +2371,9 @@ impl Module { if !self.is_indexed() { let mut path = Vec::with_capacity(4); - let mut variables = StraightHashMap::with_capacity_and_hasher( - self.variables.as_deref().map_or(0, BTreeMap::len), - Default::default(), - ); - let mut functions = StraightHashMap::with_capacity_and_hasher( - self.functions.as_ref().map_or(0, StraightHashMap::len), - Default::default(), - ); + let mut variables = new_hash_map(self.variables.as_deref().map_or(0, BTreeMap::len)); + let mut functions = + new_hash_map(self.functions.as_ref().map_or(0, StraightHashMap::len)); let mut type_iterators = BTreeMap::new(); path.push(""); From fa4096e91e2244c16a196bdc62b02f481a4e6340 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 7 Mar 2023 16:52:37 +0800 Subject: [PATCH 26/41] Fine tune Engine size. --- src/api/custom_syntax.rs | 3 +- src/api/events.rs | 2 +- src/engine.rs | 12 +++---- src/eval/data_check.rs | 12 ++++--- src/eval/debugger.rs | 2 +- src/func/call.rs | 4 +-- src/optimizer.rs | 2 +- src/tests.rs | 73 ++++++++++++++++++++-------------------- 8 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/api/custom_syntax.rs b/src/api/custom_syntax.rs index 91b89b2c..f0368b9a 100644 --- a/src/api/custom_syntax.rs +++ b/src/api/custom_syntax.rs @@ -402,7 +402,8 @@ impl Engine { parse: Box::new(parse), func: Box::new(func), scope_may_be_changed, - }, + } + .into(), ); self } diff --git a/src/api/events.rs b/src/api/events.rs index 1b7d0749..7fd2dafd 100644 --- a/src/api/events.rs +++ b/src/api/events.rs @@ -363,7 +363,7 @@ impl Engine { + SendSync + 'static, ) -> &mut Self { - self.debugger_interface = Some(Box::new((Box::new(init), Box::new(callback)))); + self.debugger_interface = Some((Box::new(init), Box::new(callback))); self } } diff --git a/src/engine.rs b/src/engine.rs index 255ab7eb..1e35d21f 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -115,7 +115,7 @@ pub struct Engine { /// Custom syntax. #[cfg(not(feature = "no_custom_syntax"))] pub(crate) custom_syntax: Option< - Box>, + Box>>, >, /// Callback closure for filtering variable definition. pub(crate) def_var_filter: Option>, @@ -147,12 +147,10 @@ pub struct Engine { /// Callback closure for debugging. #[cfg(feature = "debugging")] - pub(crate) debugger_interface: Option< - Box<( - Box, - Box, - )>, - >, + pub(crate) debugger_interface: Option<( + Box, + Box, + )>, } impl fmt::Debug for Engine { diff --git a/src/eval/data_check.rs b/src/eval/data_check.rs index e4e5dc68..f5016ea3 100644 --- a/src/eval/data_check.rs +++ b/src/eval/data_check.rs @@ -205,9 +205,13 @@ impl Engine { } // Report progress - self.progress - .as_ref() - .and_then(|p| p(global.num_operations)) - .map_or(Ok(()), |token| Err(ERR::ErrorTerminated(token, pos).into())) + if let Some(ref progress) = self.progress { + match progress(global.num_operations) { + None => Ok(()), + Some(token) => Err(ERR::ErrorTerminated(token, pos).into()), + } + } else { + Ok(()) + } } } diff --git a/src/eval/debugger.rs b/src/eval/debugger.rs index 0302bdc2..de0bdd64 100644 --- a/src/eval/debugger.rs +++ b/src/eval/debugger.rs @@ -510,7 +510,7 @@ impl Engine { let src = global.source_raw().cloned(); let src = src.as_ref().map(|s| s.as_str()); let context = EvalContext::new(self, global, caches, scope, this_ptr); - let (.., ref on_debugger) = **x; + let (.., ref on_debugger) = *x; let command = on_debugger(context, event, node, src, node.position()); diff --git a/src/func/call.rs b/src/func/call.rs index a47c3495..f28d6f08 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -464,7 +464,7 @@ impl Engine { let t = self.map_type_name(type_name::()).into(); ERR::ErrorMismatchOutputType(t, typ.into(), pos) })?; - ((print)(&text).into(), false) + (print(&text).into(), false) } else { (Dynamic::UNIT, false) } @@ -475,7 +475,7 @@ impl Engine { let t = self.map_type_name(type_name::()).into(); ERR::ErrorMismatchOutputType(t, typ.into(), pos) })?; - ((debug)(&text, global.source(), pos).into(), false) + (debug(&text, global.source(), pos).into(), false) } else { (Dynamic::UNIT, false) } diff --git a/src/optimizer.rs b/src/optimizer.rs index c2dcb0c1..021a62d0 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -1139,7 +1139,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) { .and_then(|(f, ctx)| { let context = ctx.then(|| (state.engine, x.name.as_str(), None, &state.global, *pos).into()); let (first, second) = arg_values.split_first_mut().unwrap(); - (f)(context, &mut [ first, &mut second[0] ]).ok() + f(context, &mut [ first, &mut second[0] ]).ok() }) { state.set_dirty(); *expr = Expr::from_dynamic(result, *pos); diff --git a/src/tests.rs b/src/tests.rs index e6f07dd4..de018e67 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -13,6 +13,7 @@ fn check_struct_sizes() { feature = "only_i32", any(feature = "no_float", feature = "f32_float") )); + const WORD_SIZE: usize = size_of::(); assert_eq!(size_of::(), if PACKED { 8 } else { 16 }); assert_eq!(size_of::>(), if PACKED { 8 } else { 16 }); @@ -20,10 +21,7 @@ fn check_struct_sizes() { size_of::(), if cfg!(feature = "no_position") { 0 } else { 4 } ); - assert_eq!( - size_of::(), - if IS_32_BIT { 8 } else { 16 } - ); + assert_eq!(size_of::(), 2 * WORD_SIZE); assert_eq!(size_of::(), if PACKED { 12 } else { 16 }); assert_eq!(size_of::>(), if PACKED { 12 } else { 16 }); assert_eq!(size_of::(), if IS_32_BIT { 12 } else { 16 }); @@ -34,40 +32,41 @@ fn check_struct_sizes() { #[cfg(feature = "internals")] { - assert_eq!( - size_of::(), - if IS_32_BIT { 12 } else { 24 } - ); - assert_eq!( - size_of::(), - if IS_32_BIT { 16 } else { 32 } - ); + assert_eq!(size_of::(), 3 * WORD_SIZE); + assert_eq!(size_of::(), 4 * WORD_SIZE); } - #[cfg(target_pointer_width = "64")] - { - assert_eq!(size_of::(), 536); - assert_eq!( - size_of::(), - if cfg!(feature = "no_function") { - 72 - } else { - 80 - } - ); - assert_eq!(size_of::(), 56); - assert_eq!( - size_of::(), - if cfg!(feature = "no_position") { 8 } else { 16 } - ); - assert_eq!(size_of::(), 64); - assert_eq!( - size_of::(), - if cfg!(feature = "no_position") { - 48 - } else { - 56 - } - ); + // The following only on 64-bit platforms + + if !cfg!(target_pointer_width = "64") { + return; } + + assert_eq!(size_of::(), 536); + assert_eq!( + size_of::(), + 80 - if cfg!(feature = "no_function") { + WORD_SIZE + } else { + 0 + } + ); + assert_eq!(size_of::(), 56); + assert_eq!( + size_of::(), + 16 - if cfg!(feature = "no_position") { + WORD_SIZE + } else { + 0 + } + ); + assert_eq!(size_of::(), 64); + assert_eq!( + size_of::(), + 56 - if cfg!(feature = "no_position") { + WORD_SIZE + } else { + 0 + } + ); } From 58fad030eeeed5cec7bf6d2cc2f4f7663c10d32d Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 8 Mar 2023 21:47:57 +0800 Subject: [PATCH 27/41] Do not default compares of different types. --- CHANGELOG.md | 1 + src/eval/data_check.rs | 171 ++++++++++++++++++------------------ src/eval/global_state.rs | 14 --- src/eval/mod.rs | 4 + src/func/builtin.rs | 59 +------------ src/packages/array_basic.rs | 5 +- tests/mismatched_op.rs | 5 +- tests/ops.rs | 27 +++++- 8 files changed, 125 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07b73d9d..643e1074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Bug fixes * `x += y` where `x` and `y` are `char` now works correctly. * Expressions such as `!inside` now parses correctly instead of as `!in` followed by `side`. * Custom syntax starting with symbols now works correctly and no longer raises a parse error. +* Comparing different custom types now works correctly when the appropriate comparison operators are registered. Potentially breaking changes ---------------------------- diff --git a/src/eval/data_check.rs b/src/eval/data_check.rs index f5016ea3..563881ec 100644 --- a/src/eval/data_check.rs +++ b/src/eval/data_check.rs @@ -8,90 +8,91 @@ use std::borrow::Borrow; #[cfg(feature = "no_std")] use std::prelude::v1::*; +/// Recursively calculate the sizes of an array. +/// +/// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`. +/// +/// # Panics +/// +/// Panics if any interior data is shared (should never happen). +#[cfg(not(feature = "no_index"))] +#[inline] +pub fn calc_array_sizes(array: &crate::Array) -> (usize, usize, usize) { + let (mut ax, mut mx, mut sx) = (0, 0, 0); + + for value in array { + ax += 1; + + match value.0 { + Union::Array(ref a, ..) => { + let (a, m, s) = calc_array_sizes(a); + ax += a; + mx += m; + sx += s; + } + Union::Blob(ref a, ..) => ax += 1 + a.len(), + #[cfg(not(feature = "no_object"))] + Union::Map(ref m, ..) => { + let (a, m, s) = calc_map_sizes(m); + ax += a; + mx += m; + sx += s; + } + Union::Str(ref s, ..) => sx += s.len(), + #[cfg(not(feature = "no_closure"))] + Union::Shared(..) => { + unreachable!("shared values discovered within data") + } + _ => (), + } + } + + (ax, mx, sx) +} +/// Recursively calculate the sizes of a map. +/// +/// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`. +/// +/// # Panics +/// +/// Panics if any interior data is shared (should never happen). +#[cfg(not(feature = "no_object"))] +#[inline] +pub fn calc_map_sizes(map: &crate::Map) -> (usize, usize, usize) { + let (mut ax, mut mx, mut sx) = (0, 0, 0); + + for value in map.values() { + mx += 1; + + match value.0 { + #[cfg(not(feature = "no_index"))] + Union::Array(ref a, ..) => { + let (a, m, s) = calc_array_sizes(a); + ax += a; + mx += m; + sx += s; + } + #[cfg(not(feature = "no_index"))] + Union::Blob(ref a, ..) => ax += 1 + a.len(), + Union::Map(ref m, ..) => { + let (a, m, s) = calc_map_sizes(m); + ax += a; + mx += m; + sx += s; + } + Union::Str(ref s, ..) => sx += s.len(), + #[cfg(not(feature = "no_closure"))] + Union::Shared(..) => { + unreachable!("shared values discovered within data") + } + _ => (), + } + } + + (ax, mx, sx) +} + impl Dynamic { - /// Recursively calculate the sizes of an array. - /// - /// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`. - /// - /// # Panics - /// - /// Panics if any interior data is shared (should never happen). - #[cfg(not(feature = "no_index"))] - #[inline] - pub(crate) fn calc_array_sizes(array: &crate::Array) -> (usize, usize, usize) { - let (mut ax, mut mx, mut sx) = (0, 0, 0); - - for value in array { - ax += 1; - - match value.0 { - Union::Array(ref a, ..) => { - let (a, m, s) = Self::calc_array_sizes(a); - ax += a; - mx += m; - sx += s; - } - Union::Blob(ref a, ..) => ax += 1 + a.len(), - #[cfg(not(feature = "no_object"))] - Union::Map(ref m, ..) => { - let (a, m, s) = Self::calc_map_sizes(m); - ax += a; - mx += m; - sx += s; - } - Union::Str(ref s, ..) => sx += s.len(), - #[cfg(not(feature = "no_closure"))] - Union::Shared(..) => { - unreachable!("shared values discovered within data") - } - _ => (), - } - } - - (ax, mx, sx) - } - /// Recursively calculate the sizes of a map. - /// - /// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`. - /// - /// # Panics - /// - /// Panics if any interior data is shared (should never happen). - #[cfg(not(feature = "no_object"))] - #[inline] - pub(crate) fn calc_map_sizes(map: &crate::Map) -> (usize, usize, usize) { - let (mut ax, mut mx, mut sx) = (0, 0, 0); - - for value in map.values() { - mx += 1; - - match value.0 { - #[cfg(not(feature = "no_index"))] - Union::Array(ref a, ..) => { - let (a, m, s) = Self::calc_array_sizes(a); - ax += a; - mx += m; - sx += s; - } - #[cfg(not(feature = "no_index"))] - Union::Blob(ref a, ..) => ax += 1 + a.len(), - Union::Map(ref m, ..) => { - let (a, m, s) = Self::calc_map_sizes(m); - ax += a; - mx += m; - sx += s; - } - Union::Str(ref s, ..) => sx += s.len(), - #[cfg(not(feature = "no_closure"))] - Union::Shared(..) => { - unreachable!("shared values discovered within data") - } - _ => (), - } - } - - (ax, mx, sx) - } /// Recursively calculate the sizes of a value. /// /// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`. @@ -103,11 +104,11 @@ impl Dynamic { pub(crate) fn calc_data_sizes(&self, _top: bool) -> (usize, usize, usize) { match self.0 { #[cfg(not(feature = "no_index"))] - Union::Array(ref arr, ..) => Self::calc_array_sizes(arr), + Union::Array(ref arr, ..) => calc_array_sizes(arr), #[cfg(not(feature = "no_index"))] Union::Blob(ref blob, ..) => (blob.len(), 0, 0), #[cfg(not(feature = "no_object"))] - Union::Map(ref map, ..) => Self::calc_map_sizes(map), + Union::Map(ref map, ..) => calc_map_sizes(map), Union::Str(ref s, ..) => (0, 0, s.len()), #[cfg(not(feature = "no_closure"))] Union::Shared(..) if _top => self.read_lock::().unwrap().calc_data_sizes(true), diff --git a/src/eval/global_state.rs b/src/eval/global_state.rs index 69e08c5e..caaf210f 100644 --- a/src/eval/global_state.rs +++ b/src/eval/global_state.rs @@ -127,20 +127,6 @@ impl GlobalRuntimeState { pub fn get_shared_import(&self, index: usize) -> Option { self.modules.as_ref().and_then(|m| m.get(index).cloned()) } - /// Get a mutable reference to the globally-imported [module][crate::Module] at a - /// particular index. - /// - /// Not available under `no_module`. - #[cfg(not(feature = "no_module"))] - #[allow(dead_code)] - #[inline] - #[must_use] - pub(crate) fn get_shared_import_mut( - &mut self, - index: usize, - ) -> Option<&mut crate::SharedModule> { - self.modules.as_deref_mut().and_then(|m| m.get_mut(index)) - } /// Get the index of a globally-imported [module][crate::Module] by name. /// /// Not available under `no_module`. diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 733eb5ff..443000e9 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -11,6 +11,10 @@ mod target; pub use cache::{Caches, FnResolutionCache, FnResolutionCacheEntry}; #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] pub use chaining::ChainType; +#[cfg(not(feature = "no_index"))] +pub use data_check::calc_array_sizes; +#[cfg(not(feature = "no_object"))] +pub use data_check::calc_map_sizes; #[cfg(feature = "debugging")] pub use debugger::{ BreakPoint, CallStackFrame, Debugger, DebuggerCommand, DebuggerEvent, DebuggerStatus, diff --git a/src/func/builtin.rs b/src/func/builtin.rs index 1b69ba43..d6b2a9e5 100644 --- a/src/func/builtin.rs +++ b/src/func/builtin.rs @@ -27,48 +27,6 @@ use rust_decimal::Decimal; /// The `unchecked` feature is not active. const CHECKED_BUILD: bool = cfg!(not(feature = "unchecked")); -/// Is the type a numeric type? -#[inline] -#[must_use] -fn is_numeric(type_id: TypeId) -> bool { - if type_id == TypeId::of::() { - return true; - } - - #[cfg(not(feature = "only_i64"))] - #[cfg(not(feature = "only_i32"))] - if type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - || type_id == TypeId::of::() - { - return true; - } - - #[cfg(not(feature = "only_i64"))] - #[cfg(not(feature = "only_i32"))] - #[cfg(not(target_family = "wasm"))] - if type_id == TypeId::of::() || type_id == TypeId::of::() { - return true; - } - - #[cfg(not(feature = "no_float"))] - if type_id == TypeId::of::() || type_id == TypeId::of::() { - return true; - } - - #[cfg(feature = "decimal")] - if type_id == TypeId::of::() { - return true; - } - - false -} - /// A function that returns `true`. #[inline(always)] #[allow(clippy::unnecessary_wraps)] @@ -584,22 +542,7 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option< // One of the operands is a custom type, so it is never built-in if x.is_variant() || y.is_variant() { - return if is_numeric(type1) && is_numeric(type2) { - // Disallow comparisons between different numeric types - None - } else if type1 != type2 { - // If the types are not the same, default to not compare - match op { - NotEqualsTo => Some((const_true_fn, false)), - EqualsTo | GreaterThan | GreaterThanEqualsTo | LessThan | LessThanEqualsTo => { - Some((const_false_fn, false)) - } - _ => None, - } - } else { - // Disallow comparisons between the same type - None - }; + return None; } // Default comparison operators for different types diff --git a/src/packages/array_basic.rs b/src/packages/array_basic.rs index 0e968c4c..c8cb1a86 100644 --- a/src/packages/array_basic.rs +++ b/src/packages/array_basic.rs @@ -2,9 +2,10 @@ use crate::api::deprecated::deprecated_array_functions; use crate::engine::OP_EQUALS; -use crate::eval::{calc_index, calc_offset_len}; +use crate::eval::{calc_index, calc_offset_len, calc_array_sizes}; use crate::module::ModuleFlags; use crate::plugin::*; + use crate::{ def_package, Array, Dynamic, ExclusiveRange, FnPtr, InclusiveRange, NativeCallContext, Position, RhaiResultOf, StaticVec, ERR, INT, MAX_USIZE_INT, @@ -237,7 +238,7 @@ pub mod array_functions { #[cfg(not(feature = "unchecked"))] if _ctx.engine().max_array_size() > 0 { let pad = len - array.len(); - let (a, m, s) = Dynamic::calc_array_sizes(array); + let (a, m, s) = calc_array_sizes(array); let (ax, mx, sx) = item.calc_data_sizes(true); _ctx.engine() diff --git a/tests/mismatched_op.rs b/tests/mismatched_op.rs index 7147330a..85d31857 100644 --- a/tests/mismatched_op.rs +++ b/tests/mismatched_op.rs @@ -39,7 +39,10 @@ fn test_mismatched_op_custom_type() -> Result<(), Box> { ").expect_err("should error"), EvalAltResult::ErrorFunctionNotFound(f, ..) if f == "== (TestStruct, TestStruct)")); - assert!(!engine.eval::("new_ts() == 42")?); + assert!( + matches!(*engine.eval::("new_ts() == 42").expect_err("should error"), + EvalAltResult::ErrorFunctionNotFound(f, ..) if f.starts_with("== (TestStruct, ")) + ); assert!(matches!( *engine.eval::("60 + new_ts()").expect_err("should error"), diff --git a/tests/ops.rs b/tests/ops.rs index 1077284c..ce6c5611 100644 --- a/tests/ops.rs +++ b/tests/ops.rs @@ -35,7 +35,11 @@ fn test_ops_other_number_types() -> Result<(), Box> { EvalAltResult::ErrorFunctionNotFound(f, ..) if f.starts_with("== (u16,") )); - assert!(!engine.eval_with_scope::(&mut scope, r#"x == "hello""#)?); + assert!( + matches!(*engine.eval_with_scope::(&mut scope, r#"x == "hello""#).expect_err("should error"), + EvalAltResult::ErrorFunctionNotFound(f, ..) if f.starts_with("== (u16,") + ) + ); Ok(()) } @@ -63,3 +67,24 @@ fn test_ops_precedence() -> Result<(), Box> { Ok(()) } + +#[test] +fn test_ops_custom_types() -> Result<(), Box> { + #[derive(Debug, Clone, PartialEq, Eq)] + struct Test1; + #[derive(Debug, Clone, PartialEq, Eq)] + struct Test2; + + let mut engine = Engine::new(); + + engine + .register_type_with_name::("Test1") + .register_type_with_name::("Test2") + .register_fn("new_ts1", || Test1) + .register_fn("new_ts2", || Test2) + .register_fn("==", |x: Test1, y: Test2| true); + + assert!(engine.eval::("let x = new_ts1(); let y = new_ts2(); x == y")?); + + Ok(()) +} From 8fe5bac3e9b7e21e2055717b9e2f4672b295cce6 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 8 Mar 2023 21:50:57 +0800 Subject: [PATCH 28/41] Short-circuit primary data operations. --- src/func/call.rs | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/func/call.rs b/src/func/call.rs index f28d6f08..687ed3fc 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -9,6 +9,7 @@ use crate::engine::{ }; use crate::eval::{Caches, FnResolutionCacheEntry, GlobalRuntimeState}; use crate::tokenizer::{is_valid_function_name, Token}; +use crate::types::dynamic::Union; use crate::{ calc_fn_hash, calc_fn_hash_full, Dynamic, Engine, FnArgsVec, FnPtr, ImmutableString, OptimizationLevel, Position, RhaiResult, RhaiResultOf, Scope, Shared, ERR, @@ -1596,6 +1597,77 @@ impl Engine { .0 .flatten(); + match (&lhs.0, &rhs.0) { + (Union::Unit(..), Union::Unit(..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok(Dynamic::TRUE), + Token::NotEqualsTo + | Token::GreaterThan + | Token::GreaterThanEqualsTo + | Token::LessThan + | Token::LessThanEqualsTo => return Ok(Dynamic::FALSE), + _ => (), + }, + (Union::Bool(b1, ..), Union::Bool(b2, ..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok((*b1 == *b2).into()), + Token::NotEqualsTo => return Ok((*b1 != *b2).into()), + Token::GreaterThan + | Token::GreaterThanEqualsTo + | Token::LessThan + | Token::LessThanEqualsTo => return Ok(Dynamic::FALSE), + Token::Pipe => return Ok((*b1 || *b2).into()), + Token::Ampersand => return Ok((*b1 && *b2).into()), + _ => (), + }, + (Union::Int(n1, ..), Union::Int(n2, ..)) => { + #[cfg(not(feature = "unchecked"))] + #[allow(clippy::wildcard_imports)] + use crate::packages::arithmetic::arith_basic::INT::functions::*; + + match op_token.unwrap() { + Token::EqualsTo => return Ok((*n1 == *n2).into()), + Token::NotEqualsTo => return Ok((*n1 != *n2).into()), + Token::GreaterThan => return Ok((*n1 > *n2).into()), + Token::GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), + Token::LessThan => return Ok((*n1 < *n2).into()), + Token::LessThanEqualsTo => return Ok((*n1 <= *n2).into()), + + #[cfg(not(feature = "unchecked"))] + Token::Plus => return add(*n1, *n2).map(Into::into), + #[cfg(not(feature = "unchecked"))] + Token::Minus => return subtract(*n1, *n2).map(Into::into), + #[cfg(not(feature = "unchecked"))] + Token::Multiply => return multiply(*n1, *n2).map(Into::into), + #[cfg(not(feature = "unchecked"))] + Token::Divide => return divide(*n1, *n2).map(Into::into), + #[cfg(not(feature = "unchecked"))] + Token::Modulo => return modulo(*n1, *n2).map(Into::into), + + #[cfg(feature = "unchecked")] + Token::Plus => return Ok((*n1 + *n2).into()), + #[cfg(feature = "unchecked")] + Token::Minus => return Ok((*n1 - *n2).into()), + #[cfg(feature = "unchecked")] + Token::Multiply => return Ok((*n1 * *n2).into()), + #[cfg(feature = "unchecked")] + Token::Divide => return Ok((*n1 / *n2).into()), + #[cfg(feature = "unchecked")] + Token::Modulo => return Ok((*n1 % *n2).into()), + + _ => (), + } + } + (Union::Str(s1, ..), Union::Str(s2, ..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok((s1 == s2).into()), + Token::NotEqualsTo => return Ok((s1 != s2).into()), + Token::GreaterThan => return Ok((s1 > s2).into()), + Token::GreaterThanEqualsTo => return Ok((s1 >= s2).into()), + Token::LessThan => return Ok((s1 < s2).into()), + Token::LessThanEqualsTo => return Ok((s1 <= s2).into()), + _ => (), + }, + _ => (), + } + let operands = &mut [&mut lhs, &mut rhs]; if let Some((func, need_context)) = From 80917bfad2cc6d68e2641e8c711374dc1b41b345 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 8 Mar 2023 22:03:03 +0800 Subject: [PATCH 29/41] Fix build. --- src/eval/mod.rs | 2 ++ src/func/call.rs | 31 ++++++++++++++++++------------- src/packages/array_basic.rs | 4 ++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 443000e9..814baaf2 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -11,8 +11,10 @@ mod target; pub use cache::{Caches, FnResolutionCache, FnResolutionCacheEntry}; #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] pub use chaining::ChainType; +#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "no_index"))] pub use data_check::calc_array_sizes; +#[cfg(not(feature = "unchecked"))] #[cfg(not(feature = "no_object"))] pub use data_check::calc_map_sizes; #[cfg(feature = "debugging")] diff --git a/src/func/call.rs b/src/func/call.rs index 687ed3fc..d9928fdf 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -1576,6 +1576,11 @@ impl Engine { .0 .flatten(); + match value.0 { + Union::Bool(b, ..) => return Ok((!b).into()), + _ => (), + } + return value.as_bool().map(|r| (!r).into()).or_else(|_| { let operand = &mut [&mut value]; self.exec_fn_call( @@ -1597,6 +1602,8 @@ impl Engine { .0 .flatten(); + // For extremely simple primary data operations, do it directly + // to avoid the overhead of calling a function. match (&lhs.0, &rhs.0) { (Union::Unit(..), Union::Unit(..)) => match op_token.unwrap() { Token::EqualsTo => return Ok(Dynamic::TRUE), @@ -1623,6 +1630,7 @@ impl Engine { #[allow(clippy::wildcard_imports)] use crate::packages::arithmetic::arith_basic::INT::functions::*; + #[cfg(not(feature = "unchecked"))] match op_token.unwrap() { Token::EqualsTo => return Ok((*n1 == *n2).into()), Token::NotEqualsTo => return Ok((*n1 != *n2).into()), @@ -1630,29 +1638,26 @@ impl Engine { Token::GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), Token::LessThan => return Ok((*n1 < *n2).into()), Token::LessThanEqualsTo => return Ok((*n1 <= *n2).into()), - - #[cfg(not(feature = "unchecked"))] Token::Plus => return add(*n1, *n2).map(Into::into), - #[cfg(not(feature = "unchecked"))] Token::Minus => return subtract(*n1, *n2).map(Into::into), - #[cfg(not(feature = "unchecked"))] Token::Multiply => return multiply(*n1, *n2).map(Into::into), - #[cfg(not(feature = "unchecked"))] Token::Divide => return divide(*n1, *n2).map(Into::into), - #[cfg(not(feature = "unchecked"))] Token::Modulo => return modulo(*n1, *n2).map(Into::into), - - #[cfg(feature = "unchecked")] + _ => (), + } + #[cfg(feature = "unchecked")] + match op_token.unwrap() { + Token::EqualsTo => return Ok((*n1 == *n2).into()), + Token::NotEqualsTo => return Ok((*n1 != *n2).into()), + Token::GreaterThan => return Ok((*n1 > *n2).into()), + Token::GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), + Token::LessThan => return Ok((*n1 < *n2).into()), + Token::LessThanEqualsTo => return Ok((*n1 <= *n2).into()), Token::Plus => return Ok((*n1 + *n2).into()), - #[cfg(feature = "unchecked")] Token::Minus => return Ok((*n1 - *n2).into()), - #[cfg(feature = "unchecked")] Token::Multiply => return Ok((*n1 * *n2).into()), - #[cfg(feature = "unchecked")] Token::Divide => return Ok((*n1 / *n2).into()), - #[cfg(feature = "unchecked")] Token::Modulo => return Ok((*n1 % *n2).into()), - _ => (), } } diff --git a/src/packages/array_basic.rs b/src/packages/array_basic.rs index c8cb1a86..6e2c2089 100644 --- a/src/packages/array_basic.rs +++ b/src/packages/array_basic.rs @@ -2,7 +2,7 @@ use crate::api::deprecated::deprecated_array_functions; use crate::engine::OP_EQUALS; -use crate::eval::{calc_index, calc_offset_len, calc_array_sizes}; +use crate::eval::{calc_index, calc_offset_len}; use crate::module::ModuleFlags; use crate::plugin::*; @@ -238,7 +238,7 @@ pub mod array_functions { #[cfg(not(feature = "unchecked"))] if _ctx.engine().max_array_size() > 0 { let pad = len - array.len(); - let (a, m, s) = calc_array_sizes(array); + let (a, m, s) = crate::eval::calc_array_sizes(array); let (ax, mx, sx) = item.calc_data_sizes(true); _ctx.engine() From e8dfabf96710e1808399674afb3226c52a732e5a Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 8 Mar 2023 22:48:31 +0800 Subject: [PATCH 30/41] Add floating-point. --- src/func/call.rs | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/func/call.rs b/src/func/call.rs index d9928fdf..5dc96184 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -1661,6 +1661,51 @@ impl Engine { _ => (), } } + #[cfg(not(feature = "no_float"))] + (Union::Float(f1, ..), Union::Float(f2, ..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok((**f1 == **f2).into()), + Token::NotEqualsTo => return Ok((**f1 != **f2).into()), + Token::GreaterThan => return Ok((**f1 > **f2).into()), + Token::GreaterThanEqualsTo => return Ok((**f1 >= **f2).into()), + Token::LessThan => return Ok((**f1 < **f2).into()), + Token::LessThanEqualsTo => return Ok((**f1 <= **f2).into()), + Token::Plus => return Ok((**f1 + **f2).into()), + Token::Minus => return Ok((**f1 - **f2).into()), + Token::Multiply => return Ok((**f1 * **f2).into()), + Token::Divide => return Ok((**f1 / **f2).into()), + Token::Modulo => return Ok((**f1 % **f2).into()), + _ => (), + }, + #[cfg(not(feature = "no_float"))] + (Union::Float(f1, ..), Union::Int(n2, ..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok((**f1 == (*n2 as crate::FLOAT)).into()), + Token::NotEqualsTo => return Ok((**f1 != (*n2 as crate::FLOAT)).into()), + Token::GreaterThan => return Ok((**f1 > (*n2 as crate::FLOAT)).into()), + Token::GreaterThanEqualsTo => return Ok((**f1 >= (*n2 as crate::FLOAT)).into()), + Token::LessThan => return Ok((**f1 < (*n2 as crate::FLOAT)).into()), + Token::LessThanEqualsTo => return Ok((**f1 <= (*n2 as crate::FLOAT)).into()), + Token::Plus => return Ok((**f1 + (*n2 as crate::FLOAT)).into()), + Token::Minus => return Ok((**f1 - (*n2 as crate::FLOAT)).into()), + Token::Multiply => return Ok((**f1 * (*n2 as crate::FLOAT)).into()), + Token::Divide => return Ok((**f1 / (*n2 as crate::FLOAT)).into()), + Token::Modulo => return Ok((**f1 % (*n2 as crate::FLOAT)).into()), + _ => (), + }, + #[cfg(not(feature = "no_float"))] + (Union::Int(n1, ..), Union::Float(f2, ..)) => match op_token.unwrap() { + Token::EqualsTo => return Ok(((*n1 as crate::FLOAT) == **f2).into()), + Token::NotEqualsTo => return Ok(((*n1 as crate::FLOAT) != **f2).into()), + Token::GreaterThan => return Ok(((*n1 as crate::FLOAT) > **f2).into()), + Token::GreaterThanEqualsTo => return Ok(((*n1 as crate::FLOAT) >= **f2).into()), + Token::LessThan => return Ok(((*n1 as crate::FLOAT) < **f2).into()), + Token::LessThanEqualsTo => return Ok(((*n1 as crate::FLOAT) <= **f2).into()), + Token::Plus => return Ok(((*n1 as crate::FLOAT) + **f2).into()), + Token::Minus => return Ok(((*n1 as crate::FLOAT) - **f2).into()), + Token::Multiply => return Ok(((*n1 as crate::FLOAT) * **f2).into()), + Token::Divide => return Ok(((*n1 as crate::FLOAT) / **f2).into()), + Token::Modulo => return Ok(((*n1 as crate::FLOAT) % **f2).into()), + _ => (), + }, (Union::Str(s1, ..), Union::Str(s2, ..)) => match op_token.unwrap() { Token::EqualsTo => return Ok((s1 == s2).into()), Token::NotEqualsTo => return Ok((s1 != s2).into()), From bb404a415d1f5a4c758e0af5c2dc1b6415609730 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 8 Mar 2023 22:48:36 +0800 Subject: [PATCH 31/41] Remove warnings. --- src/parser.rs | 4 ++-- src/types/scope.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 182adc61..ac621c25 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1722,7 +1722,7 @@ impl Engine { lib: &mut FnLib, settings: ParseSettings, mut lhs: Expr, - options: ChainingFlags, + _options: ChainingFlags, ) -> ParseResult { let mut settings = settings; @@ -1783,7 +1783,7 @@ impl Engine { // Disallowed module separator #[cfg(not(feature = "no_module"))] (_, token @ Token::DoubleColon) - if options.contains(ChainingFlags::DISALLOW_NAMESPACES) => + if _options.contains(ChainingFlags::DISALLOW_NAMESPACES) => { return Err(LexError::ImproperSymbol( token.literal_syntax().into(), diff --git a/src/types/scope.rs b/src/types/scope.rs index 927225b0..7d370c1e 100644 --- a/src/types/scope.rs +++ b/src/types/scope.rs @@ -392,6 +392,7 @@ impl Scope<'_> { } /// Remove the last entry from the [`Scope`] and return it. #[inline(always)] + #[allow(dead_code)] pub(crate) fn pop_entry(&mut self) -> Option<(Identifier, Dynamic, Vec)> { self.values.pop().map(|value| { ( From 906ab3a29514274fa9f689998d4cdda33e8ca6cd Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 9 Mar 2023 22:03:27 +0800 Subject: [PATCH 32/41] Add short-circuits to op-assignments. --- CHANGELOG.md | 1 + src/eval/stmt.rs | 122 ++++++++++++++++++++++++------ src/func/builtin.rs | 1 + src/func/call.rs | 180 ++++++++++++++++++++++---------------------- 4 files changed, 193 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 643e1074..812cfa50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Bug fixes * Expressions such as `!inside` now parses correctly instead of as `!in` followed by `side`. * Custom syntax starting with symbols now works correctly and no longer raises a parse error. * Comparing different custom types now works correctly when the appropriate comparison operators are registered. +* Op-assignments to bit flags or bit ranges now work correctly. Potentially breaking changes ---------------------------- diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 9a70177d..4846c9a7 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -7,6 +7,7 @@ use crate::ast::{ SwitchCasesCollection, }; use crate::func::{get_builtin_op_assignment_fn, get_hasher}; +use crate::tokenizer::Token; use crate::types::dynamic::{AccessMode, Union}; use crate::{Dynamic, Engine, RhaiResult, RhaiResultOf, Scope, ERR, INT}; use std::hash::{Hash, Hasher}; @@ -130,38 +131,117 @@ impl Engine { if let Some((hash_x, hash, op_x, op_x_str, op, op_str)) = op_info.get_op_assignment_info() { let mut lock_guard = target.write_lock::().unwrap(); - let args = &mut [&mut *lock_guard, &mut new_val]; + let mut done = false; + // Short-circuit built-in op-assignments if under Fast Operators mode if self.fast_operators() { - if let Some((func, need_context)) = - get_builtin_op_assignment_fn(op_x, args[0], args[1]) - { - // Built-in found - auto_restore! { let orig_level = global.level; global.level += 1 } + #[allow(clippy::wildcard_imports)] + use Token::*; - let context = need_context - .then(|| (self, op_x_str, global.source(), &*global, pos).into()); - return func(context, args).map(|_| ()); + done = true; + + // For extremely simple primary data operations, do it directly + // to avoid the overhead of calling a function. + match (&mut lock_guard.0, &mut new_val.0) { + (Union::Bool(b1, ..), Union::Bool(b2, ..)) => match op_x { + AndAssign => *b1 = *b1 && *b2, + OrAssign => *b1 = *b1 || *b2, + XOrAssign => *b1 = *b1 ^ *b2, + _ => done = false, + }, + (Union::Int(n1, ..), Union::Int(n2, ..)) => { + #[cfg(not(feature = "unchecked"))] + #[allow(clippy::wildcard_imports)] + use crate::packages::arithmetic::arith_basic::INT::functions::*; + + #[cfg(not(feature = "unchecked"))] + match op_x { + PlusAssign => { + *n1 = add(*n1, *n2).map_err(|err| err.fill_position(pos))? + } + MinusAssign => { + *n1 = subtract(*n1, *n2).map_err(|err| err.fill_position(pos))? + } + MultiplyAssign => { + *n1 = multiply(*n1, *n2).map_err(|err| err.fill_position(pos))? + } + DivideAssign => { + *n1 = divide(*n1, *n2).map_err(|err| err.fill_position(pos))? + } + ModuloAssign => { + *n1 = modulo(*n1, *n2).map_err(|err| err.fill_position(pos))? + } + _ => done = false, + } + #[cfg(feature = "unchecked")] + match op_x { + PlusAssign => *n1 += *n2, + MinusAssign => *n1 -= *n2, + MultiplyAssign => *n1 *= *n2, + DivideAssign => *n1 /= *n2, + ModuloAssign => *n1 %= *n2, + _ => done = false, + } + } + #[cfg(not(feature = "no_float"))] + (Union::Float(f1, ..), Union::Float(f2, ..)) => match op_x { + PlusAssign => **f1 += **f2, + MinusAssign => **f1 -= **f2, + MultiplyAssign => **f1 *= **f2, + DivideAssign => **f1 /= **f2, + ModuloAssign => **f1 %= **f2, + _ => done = false, + }, + #[cfg(not(feature = "no_float"))] + (Union::Float(f1, ..), Union::Int(n2, ..)) => match op_x { + PlusAssign => **f1 += *n2 as crate::FLOAT, + MinusAssign => **f1 -= *n2 as crate::FLOAT, + MultiplyAssign => **f1 *= *n2 as crate::FLOAT, + DivideAssign => **f1 /= *n2 as crate::FLOAT, + ModuloAssign => **f1 %= *n2 as crate::FLOAT, + _ => done = false, + }, + _ => done = false, + } + + if !done { + if let Some((func, need_context)) = + get_builtin_op_assignment_fn(op_x, &*lock_guard, &new_val) + { + // We may not need to bump the level because built-in's do not need it. + //auto_restore! { let orig_level = global.level; global.level += 1 } + + let args = &mut [&mut *lock_guard, &mut new_val]; + let context = need_context + .then(|| (self, op_x_str, global.source(), &*global, pos).into()); + let _ = func(context, args).map_err(|err| err.fill_position(pos))?; + done = true; + } } } - let opx = Some(op_x); + if !done { + let opx = Some(op_x); + let args = &mut [&mut *lock_guard, &mut new_val]; - match self.exec_native_fn_call(global, caches, op_x_str, opx, hash_x, args, true, pos) { - Ok(_) => (), - Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_x_str)) => + match self + .exec_native_fn_call(global, caches, op_x_str, opx, hash_x, args, true, pos) { - // Expand to `var = var op rhs` - let op = Some(op); + Ok(_) => (), + Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_x_str)) => + { + // Expand to `var = var op rhs` + let op = Some(op); - *args[0] = self - .exec_native_fn_call(global, caches, op_str, op, hash, args, true, pos)? - .0; + *args[0] = self + .exec_native_fn_call(global, caches, op_str, op, hash, args, true, pos)? + .0; + } + Err(err) => return Err(err), } - Err(err) => return Err(err), - } - self.check_data_size(&*args[0], root.position())?; + self.check_data_size(&*args[0], root.position())?; + } } else { // Normal assignment match target { diff --git a/src/func/builtin.rs b/src/func/builtin.rs index d6b2a9e5..50f757f5 100644 --- a/src/func/builtin.rs +++ b/src/func/builtin.rs @@ -684,6 +684,7 @@ pub fn get_builtin_op_assignment_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Opt return match op { AndAssign => impl_op!(bool = x && as_bool), OrAssign => impl_op!(bool = x || as_bool), + XOrAssign => impl_op!(bool = x ^ as_bool), _ => None, }; } diff --git a/src/func/call.rs b/src/func/call.rs index 5dc96184..079c095b 100644 --- a/src/func/call.rs +++ b/src/func/call.rs @@ -1547,6 +1547,9 @@ impl Engine { /// # Main Entry-Point (`FnCallExpr`) /// /// Evaluate a function call expression. + /// + /// This method tries to short-circuit function resolution under Fast Operators mode if the + /// function call is an operator. pub(crate) fn eval_fn_call_expr( &self, global: &mut GlobalRuntimeState, @@ -1570,28 +1573,29 @@ impl Engine { let op_token = op_token.as_ref(); // Short-circuit native unary operator call if under Fast Operators mode - if op_token == Some(&Token::Bang) && self.fast_operators() && args.len() == 1 { + if self.fast_operators() && args.len() == 1 && op_token == Some(&Token::Bang) { let mut value = self .get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])? .0 .flatten(); - match value.0 { - Union::Bool(b, ..) => return Ok((!b).into()), - _ => (), - } - - return value.as_bool().map(|r| (!r).into()).or_else(|_| { - let operand = &mut [&mut value]; - self.exec_fn_call( - global, caches, None, name, op_token, *hashes, operand, false, false, pos, - ) - .map(|(v, ..)| v) - }); + return match value.0 { + Union::Bool(b, ..) => Ok((!b).into()), + _ => { + let operand = &mut [&mut value]; + self.exec_fn_call( + global, caches, None, name, op_token, *hashes, operand, false, false, pos, + ) + .map(|(v, ..)| v) + } + }; } // Short-circuit native binary operator call if under Fast Operators mode if op_token.is_some() && self.fast_operators() && args.len() == 2 { + #[allow(clippy::wildcard_imports)] + use Token::*; + let mut lhs = self .get_arg_value(global, caches, scope, this_ptr.as_deref_mut(), &args[0])? .0 @@ -1606,23 +1610,19 @@ impl Engine { // to avoid the overhead of calling a function. match (&lhs.0, &rhs.0) { (Union::Unit(..), Union::Unit(..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok(Dynamic::TRUE), - Token::NotEqualsTo - | Token::GreaterThan - | Token::GreaterThanEqualsTo - | Token::LessThan - | Token::LessThanEqualsTo => return Ok(Dynamic::FALSE), + EqualsTo => return Ok(Dynamic::TRUE), + NotEqualsTo | GreaterThan | GreaterThanEqualsTo | LessThan + | LessThanEqualsTo => return Ok(Dynamic::FALSE), _ => (), }, (Union::Bool(b1, ..), Union::Bool(b2, ..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok((*b1 == *b2).into()), - Token::NotEqualsTo => return Ok((*b1 != *b2).into()), - Token::GreaterThan - | Token::GreaterThanEqualsTo - | Token::LessThan - | Token::LessThanEqualsTo => return Ok(Dynamic::FALSE), - Token::Pipe => return Ok((*b1 || *b2).into()), - Token::Ampersand => return Ok((*b1 && *b2).into()), + EqualsTo => return Ok((*b1 == *b2).into()), + NotEqualsTo => return Ok((*b1 != *b2).into()), + GreaterThan | GreaterThanEqualsTo | LessThan | LessThanEqualsTo => { + return Ok(Dynamic::FALSE) + } + Pipe => return Ok((*b1 || *b2).into()), + Ampersand => return Ok((*b1 && *b2).into()), _ => (), }, (Union::Int(n1, ..), Union::Int(n2, ..)) => { @@ -1632,87 +1632,87 @@ impl Engine { #[cfg(not(feature = "unchecked"))] match op_token.unwrap() { - Token::EqualsTo => return Ok((*n1 == *n2).into()), - Token::NotEqualsTo => return Ok((*n1 != *n2).into()), - Token::GreaterThan => return Ok((*n1 > *n2).into()), - Token::GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), - Token::LessThan => return Ok((*n1 < *n2).into()), - Token::LessThanEqualsTo => return Ok((*n1 <= *n2).into()), - Token::Plus => return add(*n1, *n2).map(Into::into), - Token::Minus => return subtract(*n1, *n2).map(Into::into), - Token::Multiply => return multiply(*n1, *n2).map(Into::into), - Token::Divide => return divide(*n1, *n2).map(Into::into), - Token::Modulo => return modulo(*n1, *n2).map(Into::into), + EqualsTo => return Ok((*n1 == *n2).into()), + NotEqualsTo => return Ok((*n1 != *n2).into()), + GreaterThan => return Ok((*n1 > *n2).into()), + GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), + LessThan => return Ok((*n1 < *n2).into()), + LessThanEqualsTo => return Ok((*n1 <= *n2).into()), + Plus => return add(*n1, *n2).map(Into::into), + Minus => return subtract(*n1, *n2).map(Into::into), + Multiply => return multiply(*n1, *n2).map(Into::into), + Divide => return divide(*n1, *n2).map(Into::into), + Modulo => return modulo(*n1, *n2).map(Into::into), _ => (), } #[cfg(feature = "unchecked")] match op_token.unwrap() { - Token::EqualsTo => return Ok((*n1 == *n2).into()), - Token::NotEqualsTo => return Ok((*n1 != *n2).into()), - Token::GreaterThan => return Ok((*n1 > *n2).into()), - Token::GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), - Token::LessThan => return Ok((*n1 < *n2).into()), - Token::LessThanEqualsTo => return Ok((*n1 <= *n2).into()), - Token::Plus => return Ok((*n1 + *n2).into()), - Token::Minus => return Ok((*n1 - *n2).into()), - Token::Multiply => return Ok((*n1 * *n2).into()), - Token::Divide => return Ok((*n1 / *n2).into()), - Token::Modulo => return Ok((*n1 % *n2).into()), + EqualsTo => return Ok((*n1 == *n2).into()), + NotEqualsTo => return Ok((*n1 != *n2).into()), + GreaterThan => return Ok((*n1 > *n2).into()), + GreaterThanEqualsTo => return Ok((*n1 >= *n2).into()), + LessThan => return Ok((*n1 < *n2).into()), + LessThanEqualsTo => return Ok((*n1 <= *n2).into()), + Plus => return Ok((*n1 + *n2).into()), + Minus => return Ok((*n1 - *n2).into()), + Multiply => return Ok((*n1 * *n2).into()), + Divide => return Ok((*n1 / *n2).into()), + Modulo => return Ok((*n1 % *n2).into()), _ => (), } } #[cfg(not(feature = "no_float"))] (Union::Float(f1, ..), Union::Float(f2, ..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok((**f1 == **f2).into()), - Token::NotEqualsTo => return Ok((**f1 != **f2).into()), - Token::GreaterThan => return Ok((**f1 > **f2).into()), - Token::GreaterThanEqualsTo => return Ok((**f1 >= **f2).into()), - Token::LessThan => return Ok((**f1 < **f2).into()), - Token::LessThanEqualsTo => return Ok((**f1 <= **f2).into()), - Token::Plus => return Ok((**f1 + **f2).into()), - Token::Minus => return Ok((**f1 - **f2).into()), - Token::Multiply => return Ok((**f1 * **f2).into()), - Token::Divide => return Ok((**f1 / **f2).into()), - Token::Modulo => return Ok((**f1 % **f2).into()), + EqualsTo => return Ok((**f1 == **f2).into()), + NotEqualsTo => return Ok((**f1 != **f2).into()), + GreaterThan => return Ok((**f1 > **f2).into()), + GreaterThanEqualsTo => return Ok((**f1 >= **f2).into()), + LessThan => return Ok((**f1 < **f2).into()), + LessThanEqualsTo => return Ok((**f1 <= **f2).into()), + Plus => return Ok((**f1 + **f2).into()), + Minus => return Ok((**f1 - **f2).into()), + Multiply => return Ok((**f1 * **f2).into()), + Divide => return Ok((**f1 / **f2).into()), + Modulo => return Ok((**f1 % **f2).into()), _ => (), }, #[cfg(not(feature = "no_float"))] (Union::Float(f1, ..), Union::Int(n2, ..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok((**f1 == (*n2 as crate::FLOAT)).into()), - Token::NotEqualsTo => return Ok((**f1 != (*n2 as crate::FLOAT)).into()), - Token::GreaterThan => return Ok((**f1 > (*n2 as crate::FLOAT)).into()), - Token::GreaterThanEqualsTo => return Ok((**f1 >= (*n2 as crate::FLOAT)).into()), - Token::LessThan => return Ok((**f1 < (*n2 as crate::FLOAT)).into()), - Token::LessThanEqualsTo => return Ok((**f1 <= (*n2 as crate::FLOAT)).into()), - Token::Plus => return Ok((**f1 + (*n2 as crate::FLOAT)).into()), - Token::Minus => return Ok((**f1 - (*n2 as crate::FLOAT)).into()), - Token::Multiply => return Ok((**f1 * (*n2 as crate::FLOAT)).into()), - Token::Divide => return Ok((**f1 / (*n2 as crate::FLOAT)).into()), - Token::Modulo => return Ok((**f1 % (*n2 as crate::FLOAT)).into()), + EqualsTo => return Ok((**f1 == (*n2 as crate::FLOAT)).into()), + NotEqualsTo => return Ok((**f1 != (*n2 as crate::FLOAT)).into()), + GreaterThan => return Ok((**f1 > (*n2 as crate::FLOAT)).into()), + GreaterThanEqualsTo => return Ok((**f1 >= (*n2 as crate::FLOAT)).into()), + LessThan => return Ok((**f1 < (*n2 as crate::FLOAT)).into()), + LessThanEqualsTo => return Ok((**f1 <= (*n2 as crate::FLOAT)).into()), + Plus => return Ok((**f1 + (*n2 as crate::FLOAT)).into()), + Minus => return Ok((**f1 - (*n2 as crate::FLOAT)).into()), + Multiply => return Ok((**f1 * (*n2 as crate::FLOAT)).into()), + Divide => return Ok((**f1 / (*n2 as crate::FLOAT)).into()), + Modulo => return Ok((**f1 % (*n2 as crate::FLOAT)).into()), _ => (), }, #[cfg(not(feature = "no_float"))] (Union::Int(n1, ..), Union::Float(f2, ..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok(((*n1 as crate::FLOAT) == **f2).into()), - Token::NotEqualsTo => return Ok(((*n1 as crate::FLOAT) != **f2).into()), - Token::GreaterThan => return Ok(((*n1 as crate::FLOAT) > **f2).into()), - Token::GreaterThanEqualsTo => return Ok(((*n1 as crate::FLOAT) >= **f2).into()), - Token::LessThan => return Ok(((*n1 as crate::FLOAT) < **f2).into()), - Token::LessThanEqualsTo => return Ok(((*n1 as crate::FLOAT) <= **f2).into()), - Token::Plus => return Ok(((*n1 as crate::FLOAT) + **f2).into()), - Token::Minus => return Ok(((*n1 as crate::FLOAT) - **f2).into()), - Token::Multiply => return Ok(((*n1 as crate::FLOAT) * **f2).into()), - Token::Divide => return Ok(((*n1 as crate::FLOAT) / **f2).into()), - Token::Modulo => return Ok(((*n1 as crate::FLOAT) % **f2).into()), + EqualsTo => return Ok(((*n1 as crate::FLOAT) == **f2).into()), + NotEqualsTo => return Ok(((*n1 as crate::FLOAT) != **f2).into()), + GreaterThan => return Ok(((*n1 as crate::FLOAT) > **f2).into()), + GreaterThanEqualsTo => return Ok(((*n1 as crate::FLOAT) >= **f2).into()), + LessThan => return Ok(((*n1 as crate::FLOAT) < **f2).into()), + LessThanEqualsTo => return Ok(((*n1 as crate::FLOAT) <= **f2).into()), + Plus => return Ok(((*n1 as crate::FLOAT) + **f2).into()), + Minus => return Ok(((*n1 as crate::FLOAT) - **f2).into()), + Multiply => return Ok(((*n1 as crate::FLOAT) * **f2).into()), + Divide => return Ok(((*n1 as crate::FLOAT) / **f2).into()), + Modulo => return Ok(((*n1 as crate::FLOAT) % **f2).into()), _ => (), }, (Union::Str(s1, ..), Union::Str(s2, ..)) => match op_token.unwrap() { - Token::EqualsTo => return Ok((s1 == s2).into()), - Token::NotEqualsTo => return Ok((s1 != s2).into()), - Token::GreaterThan => return Ok((s1 > s2).into()), - Token::GreaterThanEqualsTo => return Ok((s1 >= s2).into()), - Token::LessThan => return Ok((s1 < s2).into()), - Token::LessThanEqualsTo => return Ok((s1 <= s2).into()), + EqualsTo => return Ok((s1 == s2).into()), + NotEqualsTo => return Ok((s1 != s2).into()), + GreaterThan => return Ok((s1 > s2).into()), + GreaterThanEqualsTo => return Ok((s1 >= s2).into()), + LessThan => return Ok((s1 < s2).into()), + LessThanEqualsTo => return Ok((s1 <= s2).into()), _ => (), }, _ => (), @@ -1723,8 +1723,8 @@ impl Engine { if let Some((func, need_context)) = get_builtin_binary_op_fn(op_token.as_ref().unwrap(), operands[0], operands[1]) { - // Built-in found - auto_restore! { let orig_level = global.level; global.level += 1 } + // We may not need to bump the level because built-in's do not need it. + //auto_restore! { let orig_level = global.level; global.level += 1 } let context = need_context.then(|| (self, name.as_str(), None, &*global, pos).into()); From f46cbdde4977b693dff3e413133eb6a3c79be0ca Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 15 Mar 2023 08:01:44 +0800 Subject: [PATCH 33/41] Speed up parsing. --- .gitignore | 1 + CHANGELOG.md | 1 + src/tokenizer.rs | 827 ++++++++++++++++++++++++++++++++--------------- 3 files changed, 560 insertions(+), 269 deletions(-) diff --git a/.gitignore b/.gitignore index 10e86a86..63a3239a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ Rhai.toml **/*.bat doc/rhai-sync.json doc/rhai.json +tools/ .idea/ .idea .idea/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 812cfa50..ed9ad185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Enhancements * The functions `min` and `max` are added for numbers. * Range cases in `switch` statements now also match floating-point and decimal values. In order to support this, however, small numeric ranges cases are no longer unrolled. * Loading a module via `import` now gives the module access to the current scope, including variables and constants defined inside. +* Some very simple operator calls (e.g. integer add) are short-circuited to avoid the overhead of a function call, resulting in a small speed improvement. Version 1.12.0 diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2d27a948..943d2f29 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2,7 +2,7 @@ use crate::engine::{ Precedence, KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL, - KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF, + KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_TYPE_OF, }; use crate::func::native::OnParseTokenCallback; use crate::{Engine, Identifier, LexError, Position, SmartString, StaticVec, INT, UNSIGNED_INT}; @@ -308,6 +308,345 @@ impl fmt::Display for Token { } } +// Table-driven keyword recognizer generated by GNU gperf. + +const MIN_KEYWORD_LEN: usize = 1; +const MAX_KEYWORD_LEN: usize = 8; +const MIN_KEYWORD_HASH_VALUE: usize = 1; +const MAX_KEYWORD_HASH_VALUE: usize = 152; + +const KEYWORD_ASSOC_VALUES: [u8; 257] = [ + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 115, 153, 100, 153, 110, + 105, 40, 80, 2, 20, 25, 125, 95, 15, 40, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 55, + 35, 10, 5, 0, 30, 110, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 120, 105, 100, 85, 90, 153, 125, 5, + 0, 125, 35, 10, 100, 153, 20, 0, 153, 10, 0, 45, 55, 0, 153, 50, 55, 5, 0, 153, 0, 0, 35, 153, + 45, 50, 30, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 153, +]; +const KEYWORDS_LIST: [(&str, Token); 153] = [ + ("", Token::EOF), + (">", Token::GreaterThan), + (">=", Token::GreaterThanEqualsTo), + (")", Token::RightParen), + ("", Token::EOF), + ("const", Token::Const), + ("=", Token::Equals), + ("==", Token::EqualsTo), + ("continue", Token::Continue), + ("", Token::EOF), + ("catch", Token::Catch), + ("<", Token::LessThan), + ("<=", Token::LessThanEqualsTo), + ("for", Token::For), + ("loop", Token::Loop), + ("", Token::EOF), + (".", Token::Period), + ("<<", Token::LeftShift), + ("<<=", Token::LeftShiftAssign), + ("", Token::EOF), + ("false", Token::False), + ("*", Token::Multiply), + ("*=", Token::MultiplyAssign), + ("let", Token::Let), + ("", Token::EOF), + ("while", Token::While), + ("+", Token::Plus), + ("+=", Token::PlusAssign), + ("", Token::EOF), + ("", Token::EOF), + ("throw", Token::Throw), + ("}", Token::RightBrace), + (">>", Token::RightShift), + (">>=", Token::RightShiftAssign), + ("", Token::EOF), + ("", Token::EOF), + (";", Token::SemiColon), + ("=>", Token::DoubleArrow), + ("", Token::EOF), + ("else", Token::Else), + ("", Token::EOF), + ("/", Token::Divide), + ("/=", Token::DivideAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("{", Token::LeftBrace), + ("**", Token::PowerOf), + ("**=", Token::PowerOfAssign), + ("", Token::EOF), + ("", Token::EOF), + ("|", Token::Pipe), + ("|=", Token::OrAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + (":", Token::Colon), + ("..", Token::ExclusiveRange), + ("..=", Token::InclusiveRange), + ("", Token::EOF), + ("until", Token::Until), + ("switch", Token::Switch), + #[cfg(not(feature = "no_function"))] + ("private", Token::Private), + #[cfg(feature = "no_function")] + ("", Token::EOF), + ("try", Token::Try), + ("true", Token::True), + ("break", Token::Break), + ("return", Token::Return), + #[cfg(not(feature = "no_function"))] + ("fn", Token::Fn), + #[cfg(feature = "no_function")] + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + #[cfg(not(feature = "no_module"))] + ("import", Token::Import), + #[cfg(feature = "no_module")] + ("", Token::EOF), + #[cfg(not(feature = "no_object"))] + ("?.", Token::Elvis), + #[cfg(feature = "no_object")] + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + #[cfg(not(feature = "no_module"))] + ("export", Token::Export), + #[cfg(feature = "no_module")] + ("", Token::EOF), + ("in", Token::In), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("(", Token::LeftParen), + ("||", Token::Or), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("^", Token::XOr), + ("^=", Token::XOrAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("_", Token::Underscore), + ("::", Token::DoubleColon), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("-", Token::Minus), + ("-=", Token::MinusAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("]", Token::RightBracket), + ("()", Token::Unit), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("&", Token::Ampersand), + ("&=", Token::AndAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("%", Token::Modulo), + ("%=", Token::ModuloAssign), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("!", Token::Bang), + ("!=", Token::NotEqualsTo), + ("!in", Token::NotIn), + ("", Token::EOF), + ("", Token::EOF), + ("[", Token::LeftBracket), + ("if", Token::If), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + (",Token::", Token::Comma), + ("do", Token::Do), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + #[cfg(not(feature = "no_module"))] + ("as", Token::As), + #[cfg(feature = "no_module")] + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + #[cfg(not(feature = "no_index"))] + ("?[", Token::QuestionBracket), + #[cfg(feature = "no_index")] + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("??", Token::DoubleQuestion), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("&&", Token::And), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("", Token::EOF), + ("#{", Token::MapStart), +]; + +// Table-driven reserved symbol recognizer generated by GNU gperf. + +const MIN_RESERVED_LEN: usize = 1; +const MAX_RESERVED_LEN: usize = 10; +const MIN_RESERVED_HASH_VALUE: usize = 1; +const MAX_RESERVED_HASH_VALUE: usize = 112; + +const RESERVED_ASSOC_VALUES: [u8; 256] = [ + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 35, 113, 45, 25, 113, + 113, 113, 60, 55, 50, 50, 113, 15, 0, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 10, 85, 45, 5, 55, 50, 5, 113, 113, 113, 113, 113, 85, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 35, 113, 113, 113, 55, 113, 10, 40, + 5, 0, 5, 35, 10, 5, 0, 113, 113, 20, 25, 5, 45, 0, 113, 0, 0, 0, 15, 30, 20, 25, 20, 113, 113, + 20, 113, 0, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, +]; + +const RESERVED_LIST: [(&str, bool); 113] = [ + ("", false), + ("~", true), + ("is", true), + ("...", true), + ("", false), + ("print", true), + ("@", true), + ("private", cfg!(feature = "no_function")), + ("", false), + ("this", true), + ("", false), + ("thread", true), + ("as", cfg!(feature = "no_module")), + ("", false), + ("", false), + ("spawn", true), + ("static", true), + (":=", true), + ("===", true), + ("case", true), + ("super", true), + ("shared", true), + ("package", true), + ("use", true), + ("with", true), + ("curry", true), + ("$", true), + ("type_of", true), + ("nil", true), + ("sync", true), + ("yield", true), + ("import", cfg!(feature = "no_module")), + ("--", true), + ("new", true), + ("exit", true), + ("async", true), + ("export", cfg!(feature = "no_module")), + ("!.", true), + ("", false), + ("call", true), + ("match", true), + ("", false), + ("fn", cfg!(feature = "no_function")), + ("var", true), + ("null", true), + ("await", true), + ("#", true), + ("default", true), + ("!==", true), + ("eval", true), + ("debug", true), + ("?", true), + ("?.", cfg!(feature = "no_object")), + ("", false), + ("protected", true), + ("", false), + ("", false), + ("go", true), + ("", false), + ("goto", true), + ("", false), + ("public", true), + ("<-", true), + ("", false), + ("is_def_fn", cfg!(not(feature = "no_function"))), + ("is_def_var", true), + ("", false), + ("<|", true), + ("::<", true), + ("", false), + ("", false), + ("", false), + ("->", true), + ("", false), + ("", false), + ("", false), + ("module", true), + ("|>", true), + ("", false), + ("void", true), + ("", false), + ("", false), + ("#!", true), + ("", false), + ("", false), + ("", false), + ("", false), + ("?[", cfg!(feature = "no_index")), + ("", false), + ("", false), + ("", false), + ("", false), + ("Fn", true), + ("", false), + ("", false), + ("", false), + ("", false), + (":;", true), + ("", false), + ("", false), + ("", false), + ("", false), + ("++", true), + ("", false), + ("", false), + ("", false), + ("", false), + ("*)", true), + ("", false), + ("", false), + ("", false), + ("", false), + ("(*", true), +]; + impl Token { /// Is the token a literal symbol? #[must_use] @@ -529,101 +868,32 @@ impl Token { } /// Reverse lookup a symbol token from a piece of syntax. + #[inline] #[must_use] pub fn lookup_symbol_from_syntax(syntax: &str) -> Option { - #[allow(clippy::enum_glob_use)] - use Token::*; + let utf8 = syntax.as_bytes(); + let len = utf8.len(); + let mut hash_val = len; - Some(match syntax { - "{" => LeftBrace, - "}" => RightBrace, - "(" => LeftParen, - ")" => RightParen, - "[" => LeftBracket, - "]" => RightBracket, - "()" => Unit, - "+" => Plus, - "-" => Minus, - "*" => Multiply, - "/" => Divide, - ";" => SemiColon, - ":" => Colon, - "::" => DoubleColon, - "=>" => DoubleArrow, - "_" => Underscore, - "," => Comma, - "." => Period, - #[cfg(not(feature = "no_object"))] - "?." => Elvis, - "??" => DoubleQuestion, - #[cfg(not(feature = "no_index"))] - "?[" => QuestionBracket, - ".." => ExclusiveRange, - "..=" => InclusiveRange, - "#{" => MapStart, - "=" => Equals, - "true" => True, - "false" => False, - "let" => Let, - "const" => Const, - "if" => If, - "else" => Else, - "switch" => Switch, - "do" => Do, - "while" => While, - "until" => Until, - "loop" => Loop, - "for" => For, - "in" => In, - "!in" => NotIn, - "<" => LessThan, - ">" => GreaterThan, - "!" => Bang, - "<=" => LessThanEqualsTo, - ">=" => GreaterThanEqualsTo, - "==" => EqualsTo, - "!=" => NotEqualsTo, - "|" => Pipe, - "||" => Or, - "&" => Ampersand, - "&&" => And, - "continue" => Continue, - "break" => Break, - "return" => Return, - "throw" => Throw, - "try" => Try, - "catch" => Catch, - "+=" => PlusAssign, - "-=" => MinusAssign, - "*=" => MultiplyAssign, - "/=" => DivideAssign, - "<<=" => LeftShiftAssign, - ">>=" => RightShiftAssign, - "&=" => AndAssign, - "|=" => OrAssign, - "^=" => XOrAssign, - "<<" => LeftShift, - ">>" => RightShift, - "^" => XOr, - "%" => Modulo, - "%=" => ModuloAssign, - "**" => PowerOf, - "**=" => PowerOfAssign, + if !(MIN_KEYWORD_LEN..=MAX_KEYWORD_LEN).contains(&len) { + return None; + } - #[cfg(not(feature = "no_function"))] - "fn" => Fn, - #[cfg(not(feature = "no_function"))] - "private" => Private, + match len { + 1 => (), + _ => hash_val += KEYWORD_ASSOC_VALUES[(utf8[1] as usize) + 1] as usize, + } + hash_val += KEYWORD_ASSOC_VALUES[utf8[0] as usize] as usize; - #[cfg(not(feature = "no_module"))] - "import" => Import, - #[cfg(not(feature = "no_module"))] - "export" => Export, - #[cfg(not(feature = "no_module"))] - "as" => As, + if !(MIN_KEYWORD_HASH_VALUE..=MAX_KEYWORD_HASH_VALUE).contains(&hash_val) { + return None; + } - _ => return None, - }) + match KEYWORDS_LIST[hash_val] { + (_, Token::EOF) => None, + (s, ref t) if s == syntax => Some(t.clone()), + _ => None, + } } /// If another operator is after these, it's probably a unary operator @@ -1269,13 +1539,110 @@ fn get_next_token_inner( pos.advance(); let start_pos = *pos; + let cc = stream.peek_next().unwrap_or('\0'); - match (c, stream.peek_next().unwrap_or('\0')) { + // Identifiers and strings that can have non-ASCII characters + match (c, cc) { + // letter or underscore ... + #[cfg(not(feature = "unicode-xid-ident"))] + ('a'..='z' | '_' | 'A'..='Z', ..) => { + return Some(parse_identifier_token(stream, state, pos, start_pos, c)); + } + #[cfg(feature = "unicode-xid-ident")] + _ if unicode_xid::UnicodeXID::is_xid_start(c) || c == '_' => { + return Some(parse_identifier_token(stream, state, pos, start_pos, c)); + } + // " - string literal + ('"', ..) => { + return parse_string_literal(stream, state, pos, c, false, true, false) + .map_or_else( + |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), + |(result, ..)| Some((Token::StringConstant(result.into()), start_pos)), + ); + } + // ` - string literal + ('`', ..) => { + // Start from the next line if at the end of line + match stream.peek_next() { + // `\r - start from next line + Some('\r') => { + eat_next_and_advance(stream, pos); + // `\r\n + if stream.peek_next() == Some('\n') { + eat_next_and_advance(stream, pos); + } + pos.new_line(); + } + // `\n - start from next line + Some('\n') => { + eat_next_and_advance(stream, pos); + pos.new_line(); + } + _ => (), + } + + return parse_string_literal(stream, state, pos, c, true, false, true).map_or_else( + |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), + |(result, interpolated, ..)| { + if interpolated { + Some((Token::InterpolatedString(result.into()), start_pos)) + } else { + Some((Token::StringConstant(result.into()), start_pos)) + } + }, + ); + } + + // ' - character literal + ('\'', '\'') => { + return Some(( + Token::LexError(LERR::MalformedChar(String::new()).into()), + start_pos, + )) + } + ('\'', ..) => { + return Some( + parse_string_literal(stream, state, pos, c, false, false, false).map_or_else( + |(err, err_pos)| (Token::LexError(err.into()), err_pos), + |(result, ..)| { + let mut chars = result.chars(); + let first = chars.next().unwrap(); + + if chars.next().is_some() { + ( + Token::LexError(LERR::MalformedChar(result.to_string()).into()), + start_pos, + ) + } else { + (Token::CharConstant(first), start_pos) + } + }, + ), + ) + } + + _ => (), + } + + // Non-ASCII inputs are not valid here + if !c.is_ascii() || !cc.is_ascii() { + return Some(( + Token::LexError(LERR::UnexpectedInput(c.to_string()).into()), + start_pos, + )); + } + + // Match ASCII byte values (faster?) + let mut buf = [0_u8; 2]; + c.encode_utf8(&mut buf[0..1]); + cc.encode_utf8(&mut buf[1..]); + + match (buf[0], buf[1]) { // \n - ('\n', ..) => pos.new_line(), + (b'\n', ..) => pos.new_line(), // digit ... - ('0'..='9', ..) => { + (b'0'..=b'9', ..) => { let mut result = SmartString::new_const(); let mut radix_base: Option = None; let mut valid: fn(char) -> bool = is_numeric_digit; @@ -1434,117 +1801,38 @@ fn get_next_token_inner( return Some((token, num_pos)); } - // letter or underscore ... - #[cfg(not(feature = "unicode-xid-ident"))] - ('a'..='z' | '_' | 'A'..='Z', ..) => { - return Some(parse_identifier_token(stream, state, pos, start_pos, c)); - } - #[cfg(feature = "unicode-xid-ident")] - (ch, ..) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => { - return Some(parse_identifier_token(stream, state, pos, start_pos, c)); - } - - // " - string literal - ('"', ..) => { - return parse_string_literal(stream, state, pos, c, false, true, false) - .map_or_else( - |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), - |(result, ..)| Some((Token::StringConstant(result.into()), start_pos)), - ); - } - // ` - string literal - ('`', ..) => { - // Start from the next line if at the end of line - match stream.peek_next() { - // `\r - start from next line - Some('\r') => { - eat_next_and_advance(stream, pos); - // `\r\n - if stream.peek_next() == Some('\n') { - eat_next_and_advance(stream, pos); - } - pos.new_line(); - } - // `\n - start from next line - Some('\n') => { - eat_next_and_advance(stream, pos); - pos.new_line(); - } - _ => (), - } - - return parse_string_literal(stream, state, pos, c, true, false, true).map_or_else( - |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), - |(result, interpolated, ..)| { - if interpolated { - Some((Token::InterpolatedString(result.into()), start_pos)) - } else { - Some((Token::StringConstant(result.into()), start_pos)) - } - }, - ); - } - - // ' - character literal - ('\'', '\'') => { - return Some(( - Token::LexError(LERR::MalformedChar(String::new()).into()), - start_pos, - )) - } - ('\'', ..) => { - return Some( - parse_string_literal(stream, state, pos, c, false, false, false).map_or_else( - |(err, err_pos)| (Token::LexError(err.into()), err_pos), - |(result, ..)| { - let mut chars = result.chars(); - let first = chars.next().unwrap(); - - if chars.next().is_some() { - ( - Token::LexError(LERR::MalformedChar(result.to_string()).into()), - start_pos, - ) - } else { - (Token::CharConstant(first), start_pos) - } - }, - ), - ) - } - // Braces - ('{', ..) => return Some((Token::LeftBrace, start_pos)), - ('}', ..) => return Some((Token::RightBrace, start_pos)), + (b'{', ..) => return Some((Token::LeftBrace, start_pos)), + (b'}', ..) => return Some((Token::RightBrace, start_pos)), // Unit - ('(', ')') => { + (b'(', b')') => { eat_next_and_advance(stream, pos); return Some((Token::Unit, start_pos)); } // Parentheses - ('(', '*') => { + (b'(', b'*') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("(*".into())), start_pos)); } - ('(', ..) => return Some((Token::LeftParen, start_pos)), - (')', ..) => return Some((Token::RightParen, start_pos)), + (b'(', ..) => return Some((Token::LeftParen, start_pos)), + (b')', ..) => return Some((Token::RightParen, start_pos)), // Indexing - ('[', ..) => return Some((Token::LeftBracket, start_pos)), - (']', ..) => return Some((Token::RightBracket, start_pos)), + (b'[', ..) => return Some((Token::LeftBracket, start_pos)), + (b']', ..) => return Some((Token::RightBracket, start_pos)), // Map literal #[cfg(not(feature = "no_object"))] - ('#', '{') => { + (b'#', b'{') => { eat_next_and_advance(stream, pos); return Some((Token::MapStart, start_pos)); } // Shebang - ('#', '!') => return Some((Token::Reserved(Box::new("#!".into())), start_pos)), + (b'#', b'!') => return Some((Token::Reserved(Box::new("#!".into())), start_pos)), - ('#', ' ') => { + (b'#', b' ') => { eat_next_and_advance(stream, pos); let token = if stream.peek_next() == Some('{') { eat_next_and_advance(stream, pos); @@ -1555,50 +1843,50 @@ fn get_next_token_inner( return Some((Token::Reserved(Box::new(token.into())), start_pos)); } - ('#', ..) => return Some((Token::Reserved(Box::new("#".into())), start_pos)), + (b'#', ..) => return Some((Token::Reserved(Box::new("#".into())), start_pos)), // Operators - ('+', '=') => { + (b'+', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::PlusAssign, start_pos)); } - ('+', '+') => { + (b'+', b'+') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("++".into())), start_pos)); } - ('+', ..) if !state.next_token_cannot_be_unary => { + (b'+', ..) if !state.next_token_cannot_be_unary => { return Some((Token::UnaryPlus, start_pos)) } - ('+', ..) => return Some((Token::Plus, start_pos)), + (b'+', ..) => return Some((Token::Plus, start_pos)), - ('-', '0'..='9') if !state.next_token_cannot_be_unary => negated = Some(start_pos), - ('-', '0'..='9') => return Some((Token::Minus, start_pos)), - ('-', '=') => { + (b'-', b'0'..=b'9') if !state.next_token_cannot_be_unary => negated = Some(start_pos), + (b'-', b'0'..=b'9') => return Some((Token::Minus, start_pos)), + (b'-', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::MinusAssign, start_pos)); } - ('-', '>') => { + (b'-', b'>') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("->".into())), start_pos)); } - ('-', '-') => { + (b'-', b'-') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("--".into())), start_pos)); } - ('-', ..) if !state.next_token_cannot_be_unary => { + (b'-', ..) if !state.next_token_cannot_be_unary => { return Some((Token::UnaryMinus, start_pos)) } - ('-', ..) => return Some((Token::Minus, start_pos)), + (b'-', ..) => return Some((Token::Minus, start_pos)), - ('*', ')') => { + (b'*', b')') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("*)".into())), start_pos)); } - ('*', '=') => { + (b'*', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::MultiplyAssign, start_pos)); } - ('*', '*') => { + (b'*', b'*') => { eat_next_and_advance(stream, pos); return Some(( @@ -1611,10 +1899,10 @@ fn get_next_token_inner( start_pos, )); } - ('*', ..) => return Some((Token::Multiply, start_pos)), + (b'*', ..) => return Some((Token::Multiply, start_pos)), // Comments - ('/', '/') => { + (b'/', b'/') => { eat_next_and_advance(stream, pos); let mut comment: Option = match stream.peek_next() { @@ -1671,7 +1959,7 @@ fn get_next_token_inner( } } } - ('/', '*') => { + (b'/', b'*') => { state.comment_level = 1; eat_next_and_advance(stream, pos); @@ -1699,16 +1987,16 @@ fn get_next_token_inner( } } - ('/', '=') => { + (b'/', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::DivideAssign, start_pos)); } - ('/', ..) => return Some((Token::Divide, start_pos)), + (b'/', ..) => return Some((Token::Divide, start_pos)), - (';', ..) => return Some((Token::SemiColon, start_pos)), - (',', ..) => return Some((Token::Comma, start_pos)), + (b';', ..) => return Some((Token::SemiColon, start_pos)), + (b',', ..) => return Some((Token::Comma, start_pos)), - ('.', '.') => { + (b'.', b'.') => { eat_next_and_advance(stream, pos); return Some(( match stream.peek_next() { @@ -1725,9 +2013,9 @@ fn get_next_token_inner( start_pos, )); } - ('.', ..) => return Some((Token::Period, start_pos)), + (b'.', ..) => return Some((Token::Period, start_pos)), - ('=', '=') => { + (b'=', b'=') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('=') { @@ -1737,14 +2025,14 @@ fn get_next_token_inner( return Some((Token::EqualsTo, start_pos)); } - ('=', '>') => { + (b'=', b'>') => { eat_next_and_advance(stream, pos); return Some((Token::DoubleArrow, start_pos)); } - ('=', ..) => return Some((Token::Equals, start_pos)), + (b'=', ..) => return Some((Token::Equals, start_pos)), #[cfg(not(feature = "no_module"))] - (':', ':') => { + (b':', b':') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('<') { @@ -1754,25 +2042,25 @@ fn get_next_token_inner( return Some((Token::DoubleColon, start_pos)); } - (':', '=') => { + (b':', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new(":=".into())), start_pos)); } - (':', ';') => { + (b':', b';') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new(":;".into())), start_pos)); } - (':', ..) => return Some((Token::Colon, start_pos)), + (b':', ..) => return Some((Token::Colon, start_pos)), - ('<', '=') => { + (b'<', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::LessThanEqualsTo, start_pos)); } - ('<', '-') => { + (b'<', b'-') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("<-".into())), start_pos)); } - ('<', '<') => { + (b'<', b'<') => { eat_next_and_advance(stream, pos); return Some(( @@ -1785,17 +2073,17 @@ fn get_next_token_inner( start_pos, )); } - ('<', '|') => { + (b'<', b'|') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("<|".into())), start_pos)); } - ('<', ..) => return Some((Token::LessThan, start_pos)), + (b'<', ..) => return Some((Token::LessThan, start_pos)), - ('>', '=') => { + (b'>', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::GreaterThanEqualsTo, start_pos)); } - ('>', '>') => { + (b'>', b'>') => { eat_next_and_advance(stream, pos); return Some(( @@ -1808,9 +2096,9 @@ fn get_next_token_inner( start_pos, )); } - ('>', ..) => return Some((Token::GreaterThan, start_pos)), + (b'>', ..) => return Some((Token::GreaterThan, start_pos)), - ('!', 'i') => { + (b'!', b'i') => { stream.get_next().unwrap(); if stream.peek_next() == Some('n') { stream.get_next().unwrap(); @@ -1831,7 +2119,7 @@ fn get_next_token_inner( stream.unget('i'); return Some((Token::Bang, start_pos)); } - ('!', '=') => { + (b'!', b'=') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('=') { @@ -1841,55 +2129,55 @@ fn get_next_token_inner( return Some((Token::NotEqualsTo, start_pos)); } - ('!', '.') => { + (b'!', b'.') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("!.".into())), start_pos)); } - ('!', ..) => return Some((Token::Bang, start_pos)), + (b'!', ..) => return Some((Token::Bang, start_pos)), - ('|', '|') => { + (b'|', b'|') => { eat_next_and_advance(stream, pos); return Some((Token::Or, start_pos)); } - ('|', '=') => { + (b'|', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::OrAssign, start_pos)); } - ('|', '>') => { + (b'|', b'>') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("|>".into())), start_pos)); } - ('|', ..) => return Some((Token::Pipe, start_pos)), + (b'|', ..) => return Some((Token::Pipe, start_pos)), - ('&', '&') => { + (b'&', b'&') => { eat_next_and_advance(stream, pos); return Some((Token::And, start_pos)); } - ('&', '=') => { + (b'&', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::AndAssign, start_pos)); } - ('&', ..) => return Some((Token::Ampersand, start_pos)), + (b'&', ..) => return Some((Token::Ampersand, start_pos)), - ('^', '=') => { + (b'^', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::XOrAssign, start_pos)); } - ('^', ..) => return Some((Token::XOr, start_pos)), + (b'^', ..) => return Some((Token::XOr, start_pos)), - ('~', ..) => return Some((Token::Reserved(Box::new("~".into())), start_pos)), + (b'~', ..) => return Some((Token::Reserved(Box::new("~".into())), start_pos)), - ('%', '=') => { + (b'%', b'=') => { eat_next_and_advance(stream, pos); return Some((Token::ModuloAssign, start_pos)); } - ('%', ..) => return Some((Token::Modulo, start_pos)), + (b'%', ..) => return Some((Token::Modulo, start_pos)), - ('@', ..) => return Some((Token::Reserved(Box::new("@".into())), start_pos)), + (b'@', ..) => return Some((Token::Reserved(Box::new("@".into())), start_pos)), - ('$', ..) => return Some((Token::Reserved(Box::new("$".into())), start_pos)), + (b'$', ..) => return Some((Token::Reserved(Box::new("$".into())), start_pos)), - ('?', '.') => { + (b'?', b'.') => { eat_next_and_advance(stream, pos); return Some(( #[cfg(not(feature = "no_object"))] @@ -1899,11 +2187,11 @@ fn get_next_token_inner( start_pos, )); } - ('?', '?') => { + (b'?', b'?') => { eat_next_and_advance(stream, pos); return Some((Token::DoubleQuestion, start_pos)); } - ('?', '[') => { + (b'?', b'[') => { eat_next_and_advance(stream, pos); return Some(( #[cfg(not(feature = "no_index"))] @@ -1913,13 +2201,13 @@ fn get_next_token_inner( start_pos, )); } - ('?', ..) => return Some((Token::Reserved(Box::new("?".into())), start_pos)), + (b'?', ..) => return Some((Token::Reserved(Box::new("?".into())), start_pos)), - (ch, ..) if ch.is_whitespace() => (), + _ if c.is_whitespace() => (), - (ch, ..) => { + _ => { return Some(( - Token::LexError(LERR::UnexpectedInput(ch.to_string()).into()), + Token::LexError(LERR::UnexpectedInput(c.to_string()).into()), start_pos, )) } @@ -2065,32 +2353,33 @@ pub const fn is_id_continue(x: char) -> bool { /// Is a piece of syntax a reserved keyword or reserved symbol? #[must_use] pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { - match syntax { - #[cfg(feature = "no_object")] - "?." => true, - #[cfg(feature = "no_index")] - "?[" => true, - #[cfg(feature = "no_function")] - "fn" | "private" => true, - #[cfg(feature = "no_module")] - "import" | "export" | "as" => true, + let utf8 = syntax.as_bytes(); + let len = utf8.len(); + let mut hash_val = len; - // List of reserved operators - "===" | "!==" | "->" | "<-" | "?" | ":=" | ":;" | "~" | "!." | "::<" | "(*" | "*)" - | "#" | "#!" | "@" | "$" | "++" | "--" | "..." | "<|" | "|>" => true, + if !(MIN_RESERVED_LEN..=MAX_RESERVED_LEN).contains(&len) { + return false; + } - // List of reserved keywords - "public" | "protected" | "super" | "new" | "use" | "module" | "package" | "var" - | "static" | "shared" | "with" | "is" | "goto" | "exit" | "match" | "case" | "default" - | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync" | "async" | "await" - | "yield" => true, + match len { + 1 => hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize, + 2 => { + hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize; + hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize; + } + _ => { + hash_val += RESERVED_ASSOC_VALUES[utf8[2] as usize] as usize; + hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize; + hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize; + } + } - KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR - | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY | KEYWORD_THIS | KEYWORD_IS_DEF_VAR => true, - - #[cfg(not(feature = "no_function"))] - crate::engine::KEYWORD_IS_DEF_FN => true, + if !(MIN_RESERVED_HASH_VALUE..=MAX_RESERVED_HASH_VALUE).contains(&hash_val) { + return false; + } + match RESERVED_LIST[hash_val] { + (s, t) if s == syntax => t, _ => false, } } From 72508a07f5bae4ccfe63962ca4d28c5fc55ba0d4 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 15 Mar 2023 08:27:02 +0800 Subject: [PATCH 34/41] Update test output. --- codegen/ui_tests/non_clonable.stderr | 18 ++++++------- codegen/ui_tests/non_clonable_second.stderr | 18 ++++++------- .../rhai_fn_non_clonable_return.stderr | 24 ++++++++--------- .../rhai_mod_non_clonable_return.stderr | 26 +++++++++---------- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/codegen/ui_tests/non_clonable.stderr b/codegen/ui_tests/non_clonable.stderr index 83d12899..a8a642fb 100644 --- a/codegen/ui_tests/non_clonable.stderr +++ b/codegen/ui_tests/non_clonable.stderr @@ -1,14 +1,14 @@ error[E0277]: the trait bound `NonClonable: Clone` is not satisfied - --> ui_tests/non_clonable.rs:11:23 - | -11 | pub fn test_fn(input: NonClonable) -> bool { - | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` - | + --> ui_tests/non_clonable.rs:11:23 + | +11 | pub fn test_fn(input: NonClonable) -> bool { + | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` + | note: required by a bound in `rhai::Dynamic::cast` - --> $WORKSPACE/src/types/dynamic.rs - | - | pub fn cast(self) -> T { - | ^^^^^ required by this bound in `rhai::Dynamic::cast` + --> $WORKSPACE/src/types/dynamic.rs + | + | pub fn cast(self) -> T { + | ^^^^^ required by this bound in `Dynamic::cast` help: consider annotating `NonClonable` with `#[derive(Clone)]` | 3 | #[derive(Clone)] diff --git a/codegen/ui_tests/non_clonable_second.stderr b/codegen/ui_tests/non_clonable_second.stderr index 241d41d6..f764e8a8 100644 --- a/codegen/ui_tests/non_clonable_second.stderr +++ b/codegen/ui_tests/non_clonable_second.stderr @@ -1,14 +1,14 @@ error[E0277]: the trait bound `NonClonable: Clone` is not satisfied - --> ui_tests/non_clonable_second.rs:11:27 - | -11 | pub fn test_fn(a: u32, b: NonClonable) -> bool { - | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` - | + --> ui_tests/non_clonable_second.rs:11:27 + | +11 | pub fn test_fn(a: u32, b: NonClonable) -> bool { + | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` + | note: required by a bound in `rhai::Dynamic::cast` - --> $WORKSPACE/src/types/dynamic.rs - | - | pub fn cast(self) -> T { - | ^^^^^ required by this bound in `rhai::Dynamic::cast` + --> $WORKSPACE/src/types/dynamic.rs + | + | pub fn cast(self) -> T { + | ^^^^^ required by this bound in `Dynamic::cast` help: consider annotating `NonClonable` with `#[derive(Clone)]` | 3 | #[derive(Clone)] diff --git a/codegen/ui_tests/rhai_fn_non_clonable_return.stderr b/codegen/ui_tests/rhai_fn_non_clonable_return.stderr index 78260a4a..e4a8897e 100644 --- a/codegen/ui_tests/rhai_fn_non_clonable_return.stderr +++ b/codegen/ui_tests/rhai_fn_non_clonable_return.stderr @@ -1,17 +1,17 @@ error[E0277]: the trait bound `NonClonable: Clone` is not satisfied - --> ui_tests/rhai_fn_non_clonable_return.rs:11:31 - | -10 | #[export_fn] - | ------------ in this procedural macro expansion -11 | pub fn test_fn(input: f32) -> NonClonable { - | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` - | + --> ui_tests/rhai_fn_non_clonable_return.rs:11:31 + | +10 | #[export_fn] + | ------------ in this procedural macro expansion +11 | pub fn test_fn(input: f32) -> NonClonable { + | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` + | note: required by a bound in `rhai::Dynamic::from` - --> $WORKSPACE/src/types/dynamic.rs - | - | pub fn from(value: T) -> Self { - | ^^^^^ required by this bound in `rhai::Dynamic::from` - = note: this error originates in the attribute macro `export_fn` (in Nightly builds, run with -Z macro-backtrace for more info) + --> $WORKSPACE/src/types/dynamic.rs + | + | pub fn from(value: T) -> Self { + | ^^^^^ required by this bound in `Dynamic::from` + = note: this error originates in the attribute macro `export_fn` (in Nightly builds, run with -Z macro-backtrace for more info) help: consider annotating `NonClonable` with `#[derive(Clone)]` | 3 | #[derive(Clone)] diff --git a/codegen/ui_tests/rhai_mod_non_clonable_return.stderr b/codegen/ui_tests/rhai_mod_non_clonable_return.stderr index 7e646bcd..185d1678 100644 --- a/codegen/ui_tests/rhai_mod_non_clonable_return.stderr +++ b/codegen/ui_tests/rhai_mod_non_clonable_return.stderr @@ -1,18 +1,18 @@ error[E0277]: the trait bound `NonClonable: Clone` is not satisfied - --> ui_tests/rhai_mod_non_clonable_return.rs:12:35 - | -10 | #[export_module] - | ---------------- in this procedural macro expansion -11 | pub mod test_mod { -12 | pub fn test_fn(input: f32) -> NonClonable { - | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` - | + --> ui_tests/rhai_mod_non_clonable_return.rs:12:35 + | +10 | #[export_module] + | ---------------- in this procedural macro expansion +11 | pub mod test_mod { +12 | pub fn test_fn(input: f32) -> NonClonable { + | ^^^^^^^^^^^ the trait `Clone` is not implemented for `NonClonable` + | note: required by a bound in `rhai::Dynamic::from` - --> $WORKSPACE/src/types/dynamic.rs - | - | pub fn from(value: T) -> Self { - | ^^^^^ required by this bound in `rhai::Dynamic::from` - = note: this error originates in the attribute macro `export_module` (in Nightly builds, run with -Z macro-backtrace for more info) + --> $WORKSPACE/src/types/dynamic.rs + | + | pub fn from(value: T) -> Self { + | ^^^^^ required by this bound in `Dynamic::from` + = note: this error originates in the attribute macro `export_module` (in Nightly builds, run with -Z macro-backtrace for more info) help: consider annotating `NonClonable` with `#[derive(Clone)]` | 3 | #[derive(Clone)] From 2aa7b99d1e5372f1262790674a3d93eeafb01d5c Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 15 Mar 2023 08:54:34 +0800 Subject: [PATCH 35/41] Revise table-driven parsing. --- src/tokenizer.rs | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 943d2f29..5ed4bb9a 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -315,7 +315,7 @@ const MAX_KEYWORD_LEN: usize = 8; const MIN_KEYWORD_HASH_VALUE: usize = 1; const MAX_KEYWORD_HASH_VALUE: usize = 152; -const KEYWORD_ASSOC_VALUES: [u8; 257] = [ +static KEYWORD_ASSOC_VALUES: [u8; 257] = [ 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 115, 153, 100, 153, 110, 105, 40, 80, 2, 20, 25, 125, 95, 15, 40, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 55, @@ -331,7 +331,7 @@ const KEYWORD_ASSOC_VALUES: [u8; 257] = [ 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, ]; -const KEYWORDS_LIST: [(&str, Token); 153] = [ +static KEYWORDS_LIST: [(&str, Token); 153] = [ ("", Token::EOF), (">", Token::GreaterThan), (">=", Token::GreaterThanEqualsTo), @@ -515,7 +515,7 @@ const MAX_RESERVED_LEN: usize = 10; const MIN_RESERVED_HASH_VALUE: usize = 1; const MAX_RESERVED_HASH_VALUE: usize = 112; -const RESERVED_ASSOC_VALUES: [u8; 256] = [ +static RESERVED_ASSOC_VALUES: [u8; 256] = [ 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 35, 113, 45, 25, 113, 113, 113, 60, 55, 50, 50, 113, 15, 0, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, @@ -530,8 +530,7 @@ const RESERVED_ASSOC_VALUES: [u8; 256] = [ 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, ]; - -const RESERVED_LIST: [(&str, bool); 113] = [ +static RESERVED_LIST: [(&str, bool); 113] = [ ("", false), ("~", true), ("is", true), @@ -2351,27 +2350,20 @@ pub const fn is_id_continue(x: char) -> bool { } /// Is a piece of syntax a reserved keyword or reserved symbol? +#[inline] #[must_use] pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { let utf8 = syntax.as_bytes(); let len = utf8.len(); + let rounds = len.min(3); let mut hash_val = len; if !(MIN_RESERVED_LEN..=MAX_RESERVED_LEN).contains(&len) { return false; } - match len { - 1 => hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize, - 2 => { - hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize; - hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize; - } - _ => { - hash_val += RESERVED_ASSOC_VALUES[utf8[2] as usize] as usize; - hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize; - hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize; - } + for x in 0..rounds { + hash_val += RESERVED_ASSOC_VALUES[utf8[rounds - 1 - x] as usize] as usize; } if !(MIN_RESERVED_HASH_VALUE..=MAX_RESERVED_HASH_VALUE).contains(&hash_val) { From 41636eac55b5032801125d3da0033a7bd7df0e65 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 15 Mar 2023 17:22:11 +0800 Subject: [PATCH 36/41] Fine tune table-driven tokenizing. --- .gitignore | 2 +- CHANGELOG.md | 1 + src/README.md | 1 + src/api/custom_syntax.rs | 4 +- src/api/mod.rs | 2 +- src/parser.rs | 8 +- src/tokenizer.rs | 288 +++++++++++++++++++-------------------- src/tools/README.md | 7 + src/tools/keywords.txt | 102 ++++++++++++++ src/tools/reserved.txt | 93 +++++++++++++ src/types/fn_ptr.rs | 2 +- 11 files changed, 351 insertions(+), 159 deletions(-) create mode 100644 src/tools/README.md create mode 100644 src/tools/keywords.txt create mode 100644 src/tools/reserved.txt diff --git a/.gitignore b/.gitignore index 63a3239a..d2538ea1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,9 @@ benches/results clippy.toml Rhai.toml **/*.bat +**/*.exe doc/rhai-sync.json doc/rhai.json -tools/ .idea/ .idea .idea/* diff --git a/CHANGELOG.md b/CHANGELOG.md index ed9ad185..5913e007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Enhancements * Range cases in `switch` statements now also match floating-point and decimal values. In order to support this, however, small numeric ranges cases are no longer unrolled. * Loading a module via `import` now gives the module access to the current scope, including variables and constants defined inside. * Some very simple operator calls (e.g. integer add) are short-circuited to avoid the overhead of a function call, resulting in a small speed improvement. +* The tokenizer now uses table-driven keyword recognizers generated by GNU gperf. At least _theoretically_ it should be faster... Version 1.12.0 diff --git a/src/README.md b/src/README.md index fc1d7bbf..a22e5cfe 100644 --- a/src/README.md +++ b/src/README.md @@ -28,4 +28,5 @@ Sub-Directories | `func` | Support for function calls | | `eval` | Evaluation engine | | `serde` | Support for [`serde`](https://crates.io/crates/serde) | +| `tools` | External tools needed for building | | `bin` | Pre-built CLI binaries (e.g. `rhai-run`, `rhai-repl`) | diff --git a/src/api/custom_syntax.rs b/src/api/custom_syntax.rs index f0368b9a..acd01195 100644 --- a/src/api/custom_syntax.rs +++ b/src/api/custom_syntax.rs @@ -232,7 +232,7 @@ impl Engine { } let token = Token::lookup_symbol_from_syntax(s).or_else(|| { - if is_reserved_keyword_or_symbol(s) { + if is_reserved_keyword_or_symbol(s).0 { Some(Token::Reserved(Box::new(s.into()))) } else { None @@ -296,7 +296,7 @@ impl Engine { // Identifier or symbol in first position _ if segments.is_empty() - && (is_valid_identifier(s) || is_reserved_keyword_or_symbol(s)) => + && (is_valid_identifier(s) || is_reserved_keyword_or_symbol(s).0) => { // Make it a custom keyword/symbol if it is disabled or reserved if self diff --git a/src/api/mod.rs b/src/api/mod.rs index 50fbdaae..a2cf21ce 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -54,7 +54,7 @@ impl Engine { #[inline(always)] #[must_use] pub fn module_resolver(&self) -> &dyn crate::ModuleResolver { - const DUMMY_RESOLVER: crate::module::resolvers::DummyModuleResolver = + static DUMMY_RESOLVER: crate::module::resolvers::DummyModuleResolver = crate::module::resolvers::DummyModuleResolver; self.module_resolver.as_deref().unwrap_or(&DUMMY_RESOLVER) diff --git a/src/parser.rs b/src/parser.rs index ac621c25..cbe69827 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -11,7 +11,7 @@ use crate::engine::{Precedence, KEYWORD_THIS, OP_CONTAINS, OP_NOT}; use crate::eval::{Caches, GlobalRuntimeState}; use crate::func::{hashing::get_hasher, StraightHashMap}; use crate::tokenizer::{ - is_keyword_function, is_valid_function_name, is_valid_identifier, Token, TokenStream, + is_reserved_keyword_or_symbol, is_valid_function_name, is_valid_identifier, Token, TokenStream, TokenizerControl, }; use crate::types::dynamic::{AccessMode, Union}; @@ -1665,7 +1665,9 @@ impl Engine { match input.peek().expect(NEVER_ENDS).0 { // Function call is allowed to have reserved keyword - Token::LeftParen | Token::Bang | Token::Unit if is_keyword_function(&s).0 => { + Token::LeftParen | Token::Bang | Token::Unit + if is_reserved_keyword_or_symbol(&s).1 => + { Expr::Variable( (None, ns, 0, state.get_interned_string(*s)).into(), None, @@ -1824,7 +1826,7 @@ impl Engine { // Prevents capturing of the object properties as vars: xxx. state.allow_capture = false; } - (Token::Reserved(s), ..) if is_keyword_function(s).1 => (), + (Token::Reserved(s), ..) if is_reserved_keyword_or_symbol(s).2 => (), (Token::Reserved(s), pos) => { return Err(PERR::Reserved(s.to_string()).into_err(*pos)) } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 5ed4bb9a..6a15539d 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,9 +1,6 @@ //! Main module defining the lexer and parser. -use crate::engine::{ - Precedence, KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL, - KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_TYPE_OF, -}; +use crate::engine::Precedence; use crate::func::native::OnParseTokenCallback; use crate::{Engine, Identifier, LexError, Position, SmartString, StaticVec, INT, UNSIGNED_INT}; use smallvec::SmallVec; @@ -308,7 +305,9 @@ impl fmt::Display for Token { } } -// Table-driven keyword recognizer generated by GNU gperf. +// Table-driven keyword recognizer generated by GNU gperf on the file `tools/keywords.txt`. +// +// When adding new keywords, make sure to update `tools/keywords.txt` and re-generate this. const MIN_KEYWORD_LEN: usize = 1; const MAX_KEYWORD_LEN: usize = 8; @@ -508,7 +507,9 @@ static KEYWORDS_LIST: [(&str, Token); 153] = [ ("#{", Token::MapStart), ]; -// Table-driven reserved symbol recognizer generated by GNU gperf. +// Table-driven reserved symbol recognizer generated by GNU gperf on the file `tools/reserved.txt`. +// +// When adding new reserved symbols, make sure to update `tools/reserved.txt` and re-generate this. const MIN_RESERVED_LEN: usize = 1; const MAX_RESERVED_LEN: usize = 10; @@ -530,120 +531,120 @@ static RESERVED_ASSOC_VALUES: [u8; 256] = [ 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, ]; -static RESERVED_LIST: [(&str, bool); 113] = [ - ("", false), - ("~", true), - ("is", true), - ("...", true), - ("", false), - ("print", true), - ("@", true), - ("private", cfg!(feature = "no_function")), - ("", false), - ("this", true), - ("", false), - ("thread", true), - ("as", cfg!(feature = "no_module")), - ("", false), - ("", false), - ("spawn", true), - ("static", true), - (":=", true), - ("===", true), - ("case", true), - ("super", true), - ("shared", true), - ("package", true), - ("use", true), - ("with", true), - ("curry", true), - ("$", true), - ("type_of", true), - ("nil", true), - ("sync", true), - ("yield", true), - ("import", cfg!(feature = "no_module")), - ("--", true), - ("new", true), - ("exit", true), - ("async", true), - ("export", cfg!(feature = "no_module")), - ("!.", true), - ("", false), - ("call", true), - ("match", true), - ("", false), - ("fn", cfg!(feature = "no_function")), - ("var", true), - ("null", true), - ("await", true), - ("#", true), - ("default", true), - ("!==", true), - ("eval", true), - ("debug", true), - ("?", true), - ("?.", cfg!(feature = "no_object")), - ("", false), - ("protected", true), - ("", false), - ("", false), - ("go", true), - ("", false), - ("goto", true), - ("", false), - ("public", true), - ("<-", true), - ("", false), - ("is_def_fn", cfg!(not(feature = "no_function"))), - ("is_def_var", true), - ("", false), - ("<|", true), - ("::<", true), - ("", false), - ("", false), - ("", false), - ("->", true), - ("", false), - ("", false), - ("", false), - ("module", true), - ("|>", true), - ("", false), - ("void", true), - ("", false), - ("", false), - ("#!", true), - ("", false), - ("", false), - ("", false), - ("", false), - ("?[", cfg!(feature = "no_index")), - ("", false), - ("", false), - ("", false), - ("", false), - ("Fn", true), - ("", false), - ("", false), - ("", false), - ("", false), - (":;", true), - ("", false), - ("", false), - ("", false), - ("", false), - ("++", true), - ("", false), - ("", false), - ("", false), - ("", false), - ("*)", true), - ("", false), - ("", false), - ("", false), - ("", false), - ("(*", true), +static RESERVED_LIST: [(&str, bool, bool, bool); 113] = [ + ("", false, false, false), + ("~", true, false, false), + ("is", true, false, false), + ("...", true, false, false), + ("", false, false, false), + ("print", true, true, false), + ("@", true, false, false), + ("private", cfg!(feature = "no_function"), false, false), + ("", false, false, false), + ("this", true, false, false), + ("", false, false, false), + ("thread", true, false, false), + ("as", cfg!(feature = "no_module"), false, false), + ("", false, false, false), + ("", false, false, false), + ("spawn", true, false, false), + ("static", true, false, false), + (":=", true, false, false), + ("===", true, false, false), + ("case", true, false, false), + ("super", true, false, false), + ("shared", true, false, false), + ("package", true, false, false), + ("use", true, false, false), + ("with", true, false, false), + ("curry", true, true, true), + ("$", true, false, false), + ("type_of", true, true, true), + ("nil", true, false, false), + ("sync", true, false, false), + ("yield", true, false, false), + ("import", cfg!(feature = "no_module"), false, false), + ("--", true, false, false), + ("new", true, false, false), + ("exit", true, false, false), + ("async", true, false, false), + ("export", cfg!(feature = "no_module"), false, false), + ("!.", true, false, false), + ("", false, false, false), + ("call", true, true, true), + ("match", true, false, false), + ("", false, false, false), + ("fn", cfg!(feature = "no_function"), false, false), + ("var", true, false, false), + ("null", true, false, false), + ("await", true, false, false), + ("#", true, false, false), + ("default", true, false, false), + ("!==", true, false, false), + ("eval", true, true, false), + ("debug", true, true, false), + ("?", true, false, false), + ("?.", cfg!(feature = "no_object"), false, false), + ("", false, false, false), + ("protected", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("go", true, false, false), + ("", false, false, false), + ("goto", true, false, false), + ("", false, false, false), + ("public", true, false, false), + ("<-", true, false, false), + ("", false, false, false), + ("is_def_fn", cfg!(not(feature = "no_function")), true, false), + ("is_def_var", true, true, false), + ("", false, false, false), + ("<|", true, false, false), + ("::<", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("->", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("module", true, false, false), + ("|>", true, false, false), + ("", false, false, false), + ("void", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("#!", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("?[", cfg!(feature = "no_index"), false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("Fn", true, true, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + (":;", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("++", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("*)", true, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("", false, false, false), + ("(*", true, false, false), ]; impl Token { @@ -2250,7 +2251,7 @@ fn parse_identifier_token( return (token, start_pos); } - if is_reserved_keyword_or_symbol(&identifier) { + if is_reserved_keyword_or_symbol(&identifier).0 { return (Token::Reserved(Box::new(identifier)), start_pos); } @@ -2264,30 +2265,6 @@ fn parse_identifier_token( (Token::Identifier(identifier.into()), start_pos) } -/// Can a keyword be called like a function? -/// -/// # Return values -/// -/// The first `bool` indicates whether the keyword can be called normally as a function. -/// -/// The second `bool` indicates whether the keyword can be called in method-call style. -#[inline] -#[must_use] -pub fn is_keyword_function(name: &str) -> (bool, bool) { - match name { - KEYWORD_TYPE_OF | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY => (true, true), - - KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_EVAL | KEYWORD_FN_PTR | KEYWORD_IS_DEF_VAR => { - (true, false) - } - - #[cfg(not(feature = "no_function"))] - crate::engine::KEYWORD_IS_DEF_FN => (true, false), - - _ => (false, false), - } -} - /// _(internals)_ Is a text string a valid identifier? /// Exported under the `internals` feature only. #[must_use] @@ -2313,7 +2290,7 @@ pub fn is_valid_identifier(name: &str) -> bool { #[must_use] pub fn is_valid_function_name(name: &str) -> bool { is_valid_identifier(name) - && !is_reserved_keyword_or_symbol(name) + && !is_reserved_keyword_or_symbol(name).0 && Token::lookup_symbol_from_syntax(name).is_none() } @@ -2350,16 +2327,24 @@ pub const fn is_id_continue(x: char) -> bool { } /// Is a piece of syntax a reserved keyword or reserved symbol? +/// +/// # Return values +/// +/// The first `bool` indicates whether it is a reserved keyword or symbol. +/// +/// The second `bool` indicates whether the keyword can be called normally as a function. +/// +/// The third `bool` indicates whether the keyword can be called in method-call style. #[inline] #[must_use] -pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { +pub fn is_reserved_keyword_or_symbol(syntax: &str) -> (bool, bool, bool) { let utf8 = syntax.as_bytes(); let len = utf8.len(); let rounds = len.min(3); let mut hash_val = len; if !(MIN_RESERVED_LEN..=MAX_RESERVED_LEN).contains(&len) { - return false; + return (false, false, false); } for x in 0..rounds { @@ -2367,12 +2352,13 @@ pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { } if !(MIN_RESERVED_HASH_VALUE..=MAX_RESERVED_HASH_VALUE).contains(&hash_val) { - return false; + return (false, false, false); } match RESERVED_LIST[hash_val] { - (s, t) if s == syntax => t, - _ => false, + ("", ..) => (false, false, false), + (s, true, a, b) => (s == syntax, a, b), + _ => (false, false, false), } } diff --git a/src/tools/README.md b/src/tools/README.md new file mode 100644 index 00000000..ff8fe844 --- /dev/null +++ b/src/tools/README.md @@ -0,0 +1,7 @@ +Build Tools +=========== + +| File | Description | +| -------------- | ------------------------------------------- | +| `keywords.txt` | Input file for GNU gperf for the tokenizer. | +| `reserved.txt` | Input file for GNU gperf for the tokenizer. | diff --git a/src/tools/keywords.txt b/src/tools/keywords.txt new file mode 100644 index 00000000..33375c57 --- /dev/null +++ b/src/tools/keywords.txt @@ -0,0 +1,102 @@ +// This file holds a list of keywords/symbols for the Rhai language, with mapping to +// an appropriate `Token` variant. +// +// Generate the output table via: +// ```bash +// gperf -t keywords.txt +// ``` +// +// Since GNU gperf does not produce Rust output, the ANSI-C output must be hand-edited and +// manually spliced into `tokenizer.rs`. +// +// This includes: +// * Rewrite the C hashing program (especially since it uses a `switch` statement with fall-through) +// into equivalent Rust as the function `lookup_symbol_from_syntax`. +// * Update the values for the `???_KEYWORD_???` constants. +// * Copy the `asso_values` array into `KEYWORD_ASSOC_VALUES`. +// * Copy the `wordlist` array into `KEYWORDS_LIST` with the following modifications: +// - Remove the `#line` comments +// - Change the entry wrapping `{ .. }` into tuples `( .. )` +// - Replace all entries `("")` by `("", Token::EOF)` +// - Put feature flags on the appropriate lines, and duplicating lines that maps to `Token::EOF` +// for the opposite feature flags +// +struct keyword; +%% +"{", Token::LeftBrace +"}", Token::RightBrace +"(", Token::LeftParen +")", Token::RightParen +"[", Token::LeftBracket +"]", Token::RightBracket +"()", Token::Unit +"+", Token::Plus +"-", Token::Minus +"*", Token::Multiply +"/", Token::Divide +";", Token::SemiColon +":", Token::Colon +"::", Token::DoubleColon +"=>", Token::DoubleArrow +"_", Token::Underscore +",", Token::Comma +".", Token::Period +"?.", Token::Elvis +"??", Token::DoubleQuestion +"?[", Token::QuestionBracket +"..", Token::ExclusiveRange +"..=", Token::InclusiveRange +"#{", Token::MapStart +"=", Token::Equals +"true", Token::True +"false", Token::False +"let", Token::Let +"const", Token::Const +"if", Token::If +"else", Token::Else +"switch", Token::Switch +"do", Token::Do +"while", Token::While +"until", Token::Until +"loop", Token::Loop +"for", Token::For +"in", Token::In +"!in", Token::NotIn +"<", Token::LessThan +">", Token::GreaterThan +"<=", Token::LessThanEqualsTo +">=", Token::GreaterThanEqualsTo +"==", Token::EqualsTo +"!=", Token::NotEqualsTo +"!", Token::Bang +"|", Token::Pipe +"||", Token::Or +"&", Token::Ampersand +"&&", Token::And +"continue", Token::Continue +"break", Token::Break +"return", Token::Return +"throw", Token::Throw +"try", Token::Try +"catch", Token::Catch +"+=", Token::PlusAssign +"-=", Token::MinusAssign +"*=", Token::MultiplyAssign +"/=", Token::DivideAssign +"<<=", Token::LeftShiftAssign +">>=", Token::RightShiftAssign +"&=", Token::AndAssign +"|=", Token::OrAssign +"^=", Token::XOrAssign +"<<", Token::LeftShift +">>", Token::RightShift +"^", Token::XOr +"%", Token::Modulo +"%=", Token::ModuloAssign +"**", Token::PowerOf +"**=", Token::PowerOfAssign +"fn", Token::Fn +"private", Token::Private +"import", Token::Import +"export", Token::Export +"as", Token::As diff --git a/src/tools/reserved.txt b/src/tools/reserved.txt new file mode 100644 index 00000000..2dbe79cd --- /dev/null +++ b/src/tools/reserved.txt @@ -0,0 +1,93 @@ +// This file holds a list of reserved symbols for the Rhai language. +// +// The mapped attributes are: +// - is this a reserved symbol? (bool) +// - can this keyword be called normally as a function? (bool) +// - can this keyword be called in method-call style? (bool) +// +// Generate the output table via: +// ```bash +// gperf -t reserved.txt +// ``` +// +// Since GNU gperf does not produce Rust output, the ANSI-C output must be hand-edited and +// manually spliced into `tokenizer.rs`. +// +// This includes: +// * Rewrite the C hashing program (especially since it uses a `switch` statement with fall-through) +// into equivalent Rust as the function `is_reserved_keyword_or_symbol`. +// * Update the values for the `???_RESERVED_???` constants. +// * Copy the `asso_values` array into `RESERVED_ASSOC_VALUES`. +// * Copy the `wordlist` array into `RESERVED_LIST` with the following modifications: +// - Remove the `#line` comments +// - Change the entry wrapping `{ .. }` into tuples `( .. )` +// - Replace all entries `("")` by `("", false, false, false)` +// - Feature flags can be incorporated directly into the output via the `cfg!` macro +// +struct reserved; +%% +"?.", cfg!(feature = "no_object"), false, false +"?[", cfg!(feature = "no_index"), false, false +"fn", cfg!(feature = "no_function"), false, false +"private", cfg!(feature = "no_function"), false, false +"import", cfg!(feature = "no_module"), false, false +"export", cfg!(feature = "no_module"), false, false +"as", cfg!(feature = "no_module"), false, false +"===", true, false, false +"!==", true, false, false +"->", true, false, false +"<-", true, false, false +"?", true, false, false +":=", true, false, false +":;", true, false, false +"~", true, false, false +"!.", true, false, false +"::<", true, false, false +"(*", true, false, false +"*)", true, false, false +"#", true, false, false +"#!", true, false, false +"@", true, false, false +"$", true, false, false +"++", true, false, false +"--", true, false, false +"...", true, false, false +"<|", true, false, false +"|>", true, false, false +"public", true, false, false +"protected", true, false, false +"super", true, false, false +"new", true, false, false +"use", true, false, false +"module", true, false, false +"package", true, false, false +"var", true, false, false +"static", true, false, false +"shared", true, false, false +"with", true, false, false +"is", true, false, false +"goto", true, false, false +"exit", true, false, false +"match", true, false, false +"case", true, false, false +"default", true, false, false +"void", true, false, false +"null", true, false, false +"nil", true, false, false +"spawn", true, false, false +"thread", true, false, false +"go", true, false, false +"sync", true, false, false +"async", true, false, false +"await", true, false, false +"yield", true, false, false +"print", true, true, false +"debug", true, true, false +"type_of", true, true, true +"eval", true, true, false +"Fn", true, true, false +"call", true, true, true +"curry", true, true, true +"this", true, false, false +"is_def_var", true, true, false +"is_def_fn", cfg!(not(feature = "no_function")), true, false diff --git a/src/types/fn_ptr.rs b/src/types/fn_ptr.rs index b73246a7..864aea58 100644 --- a/src/types/fn_ptr.rs +++ b/src/types/fn_ptr.rs @@ -538,7 +538,7 @@ impl TryFrom for FnPtr { #[cfg(not(feature = "no_function"))] fn_def: None, }) - } else if is_reserved_keyword_or_symbol(&value) + } else if is_reserved_keyword_or_symbol(&value).0 || Token::lookup_symbol_from_syntax(&value).is_some() { Err( From 952c77d9bbb2b50276ad423e3214c2bcd698d9a7 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 15 Mar 2023 18:31:53 +0800 Subject: [PATCH 37/41] Fix feature. --- src/tokenizer.rs | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6a15539d..6b91af14 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1544,12 +1544,7 @@ fn get_next_token_inner( // Identifiers and strings that can have non-ASCII characters match (c, cc) { // letter or underscore ... - #[cfg(not(feature = "unicode-xid-ident"))] - ('a'..='z' | '_' | 'A'..='Z', ..) => { - return Some(parse_identifier_token(stream, state, pos, start_pos, c)); - } - #[cfg(feature = "unicode-xid-ident")] - _ if unicode_xid::UnicodeXID::is_xid_start(c) || c == '_' => { + _ if is_id_first_alphabetic(c) || c == '_' => { return Some(parse_identifier_token(stream, state, pos, start_pos, c)); } // " - string literal @@ -1625,7 +1620,7 @@ fn get_next_token_inner( } // Non-ASCII inputs are not valid here - if !c.is_ascii() || !cc.is_ascii() { + if !c.is_ascii() { return Some(( Token::LexError(LERR::UnexpectedInput(c.to_string()).into()), start_pos, @@ -1635,7 +1630,9 @@ fn get_next_token_inner( // Match ASCII byte values (faster?) let mut buf = [0_u8; 2]; c.encode_utf8(&mut buf[0..1]); - cc.encode_utf8(&mut buf[1..]); + if cc.is_ascii() { + cc.encode_utf8(&mut buf[1..]); + } match (buf[0], buf[1]) { // \n @@ -2295,35 +2292,23 @@ pub fn is_valid_function_name(name: &str) -> bool { } /// Is a character valid to start an identifier? -#[cfg(feature = "unicode-xid-ident")] #[inline(always)] #[must_use] pub fn is_id_first_alphabetic(x: char) -> bool { - unicode_xid::UnicodeXID::is_xid_start(x) + #[cfg(feature = "unicode-xid-ident")] + return unicode_xid::UnicodeXID::is_xid_start(x); + #[cfg(not(feature = "unicode-xid-ident"))] + return x.is_ascii_alphabetic(); } /// Is a character valid for an identifier? -#[cfg(feature = "unicode-xid-ident")] #[inline(always)] #[must_use] pub fn is_id_continue(x: char) -> bool { - unicode_xid::UnicodeXID::is_xid_continue(x) -} - -/// Is a character valid to start an identifier? -#[cfg(not(feature = "unicode-xid-ident"))] -#[inline(always)] -#[must_use] -pub const fn is_id_first_alphabetic(x: char) -> bool { - x.is_ascii_alphabetic() -} - -/// Is a character valid for an identifier? -#[cfg(not(feature = "unicode-xid-ident"))] -#[inline(always)] -#[must_use] -pub const fn is_id_continue(x: char) -> bool { - x.is_ascii_alphanumeric() || x == '_' + #[cfg(feature = "unicode-xid-ident")] + return unicode_xid::UnicodeXID::is_xid_continue(x); + #[cfg(not(feature = "unicode-xid-ident"))] + return x.is_ascii_alphanumeric() || x == '_'; } /// Is a piece of syntax a reserved keyword or reserved symbol? From 55f022174b0eb0c54cb5837267d45d62434710f5 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 16 Mar 2023 13:05:29 +0800 Subject: [PATCH 38/41] Remove matching by ASCII because compiler should already optimize. --- src/tokenizer.rs | 332 +++++++++++++++++++++++------------------------ 1 file changed, 164 insertions(+), 168 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6b91af14..07ad2f9e 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -871,6 +871,8 @@ impl Token { #[inline] #[must_use] pub fn lookup_symbol_from_syntax(syntax: &str) -> Option { + // This implementation is based upon a pre-calculated table generated + // by GNU gperf on the list of keywords. let utf8 = syntax.as_bytes(); let len = utf8.len(); let mut hash_val = len; @@ -891,7 +893,11 @@ impl Token { match KEYWORDS_LIST[hash_val] { (_, Token::EOF) => None, - (s, ref t) if s == syntax => Some(t.clone()), + // Fail early to avoid calling memcmp() + // Since we are already working with bytes, mind as well check the first one + (s, ref t) if s.len() == len && s.as_bytes()[0] == utf8[0] && s == syntax => { + Some(t.clone()) + } _ => None, } } @@ -1543,103 +1549,11 @@ fn get_next_token_inner( // Identifiers and strings that can have non-ASCII characters match (c, cc) { - // letter or underscore ... - _ if is_id_first_alphabetic(c) || c == '_' => { - return Some(parse_identifier_token(stream, state, pos, start_pos, c)); - } - // " - string literal - ('"', ..) => { - return parse_string_literal(stream, state, pos, c, false, true, false) - .map_or_else( - |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), - |(result, ..)| Some((Token::StringConstant(result.into()), start_pos)), - ); - } - // ` - string literal - ('`', ..) => { - // Start from the next line if at the end of line - match stream.peek_next() { - // `\r - start from next line - Some('\r') => { - eat_next_and_advance(stream, pos); - // `\r\n - if stream.peek_next() == Some('\n') { - eat_next_and_advance(stream, pos); - } - pos.new_line(); - } - // `\n - start from next line - Some('\n') => { - eat_next_and_advance(stream, pos); - pos.new_line(); - } - _ => (), - } - - return parse_string_literal(stream, state, pos, c, true, false, true).map_or_else( - |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), - |(result, interpolated, ..)| { - if interpolated { - Some((Token::InterpolatedString(result.into()), start_pos)) - } else { - Some((Token::StringConstant(result.into()), start_pos)) - } - }, - ); - } - - // ' - character literal - ('\'', '\'') => { - return Some(( - Token::LexError(LERR::MalformedChar(String::new()).into()), - start_pos, - )) - } - ('\'', ..) => { - return Some( - parse_string_literal(stream, state, pos, c, false, false, false).map_or_else( - |(err, err_pos)| (Token::LexError(err.into()), err_pos), - |(result, ..)| { - let mut chars = result.chars(); - let first = chars.next().unwrap(); - - if chars.next().is_some() { - ( - Token::LexError(LERR::MalformedChar(result.to_string()).into()), - start_pos, - ) - } else { - (Token::CharConstant(first), start_pos) - } - }, - ), - ) - } - - _ => (), - } - - // Non-ASCII inputs are not valid here - if !c.is_ascii() { - return Some(( - Token::LexError(LERR::UnexpectedInput(c.to_string()).into()), - start_pos, - )); - } - - // Match ASCII byte values (faster?) - let mut buf = [0_u8; 2]; - c.encode_utf8(&mut buf[0..1]); - if cc.is_ascii() { - cc.encode_utf8(&mut buf[1..]); - } - - match (buf[0], buf[1]) { // \n - (b'\n', ..) => pos.new_line(), + ('\n', ..) => pos.new_line(), // digit ... - (b'0'..=b'9', ..) => { + ('0'..='9', ..) => { let mut result = SmartString::new_const(); let mut radix_base: Option = None; let mut valid: fn(char) -> bool = is_numeric_digit; @@ -1798,38 +1712,107 @@ fn get_next_token_inner( return Some((token, num_pos)); } + // " - string literal + ('"', ..) => { + return parse_string_literal(stream, state, pos, c, false, true, false) + .map_or_else( + |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), + |(result, ..)| Some((Token::StringConstant(result.into()), start_pos)), + ); + } + // ` - string literal + ('`', ..) => { + // Start from the next line if at the end of line + match stream.peek_next() { + // `\r - start from next line + Some('\r') => { + eat_next_and_advance(stream, pos); + // `\r\n + if stream.peek_next() == Some('\n') { + eat_next_and_advance(stream, pos); + } + pos.new_line(); + } + // `\n - start from next line + Some('\n') => { + eat_next_and_advance(stream, pos); + pos.new_line(); + } + _ => (), + } + + return parse_string_literal(stream, state, pos, c, true, false, true).map_or_else( + |(err, err_pos)| Some((Token::LexError(err.into()), err_pos)), + |(result, interpolated, ..)| { + if interpolated { + Some((Token::InterpolatedString(result.into()), start_pos)) + } else { + Some((Token::StringConstant(result.into()), start_pos)) + } + }, + ); + } + + // ' - character literal + ('\'', '\'') => { + return Some(( + Token::LexError(LERR::MalformedChar(String::new()).into()), + start_pos, + )) + } + ('\'', ..) => { + return Some( + parse_string_literal(stream, state, pos, c, false, false, false).map_or_else( + |(err, err_pos)| (Token::LexError(err.into()), err_pos), + |(result, ..)| { + let mut chars = result.chars(); + let first = chars.next().unwrap(); + + if chars.next().is_some() { + ( + Token::LexError(LERR::MalformedChar(result.to_string()).into()), + start_pos, + ) + } else { + (Token::CharConstant(first), start_pos) + } + }, + ), + ) + } + // Braces - (b'{', ..) => return Some((Token::LeftBrace, start_pos)), - (b'}', ..) => return Some((Token::RightBrace, start_pos)), + ('{', ..) => return Some((Token::LeftBrace, start_pos)), + ('}', ..) => return Some((Token::RightBrace, start_pos)), // Unit - (b'(', b')') => { + ('(', ')') => { eat_next_and_advance(stream, pos); return Some((Token::Unit, start_pos)); } // Parentheses - (b'(', b'*') => { + ('(', '*') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("(*".into())), start_pos)); } - (b'(', ..) => return Some((Token::LeftParen, start_pos)), - (b')', ..) => return Some((Token::RightParen, start_pos)), + ('(', ..) => return Some((Token::LeftParen, start_pos)), + (')', ..) => return Some((Token::RightParen, start_pos)), // Indexing - (b'[', ..) => return Some((Token::LeftBracket, start_pos)), - (b']', ..) => return Some((Token::RightBracket, start_pos)), + ('[', ..) => return Some((Token::LeftBracket, start_pos)), + (']', ..) => return Some((Token::RightBracket, start_pos)), // Map literal #[cfg(not(feature = "no_object"))] - (b'#', b'{') => { + ('#', '{') => { eat_next_and_advance(stream, pos); return Some((Token::MapStart, start_pos)); } // Shebang - (b'#', b'!') => return Some((Token::Reserved(Box::new("#!".into())), start_pos)), + ('#', '!') => return Some((Token::Reserved(Box::new("#!".into())), start_pos)), - (b'#', b' ') => { + ('#', ' ') => { eat_next_and_advance(stream, pos); let token = if stream.peek_next() == Some('{') { eat_next_and_advance(stream, pos); @@ -1840,50 +1823,50 @@ fn get_next_token_inner( return Some((Token::Reserved(Box::new(token.into())), start_pos)); } - (b'#', ..) => return Some((Token::Reserved(Box::new("#".into())), start_pos)), + ('#', ..) => return Some((Token::Reserved(Box::new("#".into())), start_pos)), // Operators - (b'+', b'=') => { + ('+', '=') => { eat_next_and_advance(stream, pos); return Some((Token::PlusAssign, start_pos)); } - (b'+', b'+') => { + ('+', '+') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("++".into())), start_pos)); } - (b'+', ..) if !state.next_token_cannot_be_unary => { + ('+', ..) if !state.next_token_cannot_be_unary => { return Some((Token::UnaryPlus, start_pos)) } - (b'+', ..) => return Some((Token::Plus, start_pos)), + ('+', ..) => return Some((Token::Plus, start_pos)), - (b'-', b'0'..=b'9') if !state.next_token_cannot_be_unary => negated = Some(start_pos), - (b'-', b'0'..=b'9') => return Some((Token::Minus, start_pos)), - (b'-', b'=') => { + ('-', '0'..='9') if !state.next_token_cannot_be_unary => negated = Some(start_pos), + ('-', '0'..='9') => return Some((Token::Minus, start_pos)), + ('-', '=') => { eat_next_and_advance(stream, pos); return Some((Token::MinusAssign, start_pos)); } - (b'-', b'>') => { + ('-', '>') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("->".into())), start_pos)); } - (b'-', b'-') => { + ('-', '-') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("--".into())), start_pos)); } - (b'-', ..) if !state.next_token_cannot_be_unary => { + ('-', ..) if !state.next_token_cannot_be_unary => { return Some((Token::UnaryMinus, start_pos)) } - (b'-', ..) => return Some((Token::Minus, start_pos)), + ('-', ..) => return Some((Token::Minus, start_pos)), - (b'*', b')') => { + ('*', ')') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("*)".into())), start_pos)); } - (b'*', b'=') => { + ('*', '=') => { eat_next_and_advance(stream, pos); return Some((Token::MultiplyAssign, start_pos)); } - (b'*', b'*') => { + ('*', '*') => { eat_next_and_advance(stream, pos); return Some(( @@ -1896,10 +1879,10 @@ fn get_next_token_inner( start_pos, )); } - (b'*', ..) => return Some((Token::Multiply, start_pos)), + ('*', ..) => return Some((Token::Multiply, start_pos)), // Comments - (b'/', b'/') => { + ('/', '/') => { eat_next_and_advance(stream, pos); let mut comment: Option = match stream.peek_next() { @@ -1956,7 +1939,7 @@ fn get_next_token_inner( } } } - (b'/', b'*') => { + ('/', '*') => { state.comment_level = 1; eat_next_and_advance(stream, pos); @@ -1984,16 +1967,16 @@ fn get_next_token_inner( } } - (b'/', b'=') => { + ('/', '=') => { eat_next_and_advance(stream, pos); return Some((Token::DivideAssign, start_pos)); } - (b'/', ..) => return Some((Token::Divide, start_pos)), + ('/', ..) => return Some((Token::Divide, start_pos)), - (b';', ..) => return Some((Token::SemiColon, start_pos)), - (b',', ..) => return Some((Token::Comma, start_pos)), + (';', ..) => return Some((Token::SemiColon, start_pos)), + (',', ..) => return Some((Token::Comma, start_pos)), - (b'.', b'.') => { + ('.', '.') => { eat_next_and_advance(stream, pos); return Some(( match stream.peek_next() { @@ -2010,9 +1993,9 @@ fn get_next_token_inner( start_pos, )); } - (b'.', ..) => return Some((Token::Period, start_pos)), + ('.', ..) => return Some((Token::Period, start_pos)), - (b'=', b'=') => { + ('=', '=') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('=') { @@ -2022,14 +2005,14 @@ fn get_next_token_inner( return Some((Token::EqualsTo, start_pos)); } - (b'=', b'>') => { + ('=', '>') => { eat_next_and_advance(stream, pos); return Some((Token::DoubleArrow, start_pos)); } - (b'=', ..) => return Some((Token::Equals, start_pos)), + ('=', ..) => return Some((Token::Equals, start_pos)), #[cfg(not(feature = "no_module"))] - (b':', b':') => { + (':', ':') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('<') { @@ -2039,25 +2022,25 @@ fn get_next_token_inner( return Some((Token::DoubleColon, start_pos)); } - (b':', b'=') => { + (':', '=') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new(":=".into())), start_pos)); } - (b':', b';') => { + (':', ';') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new(":;".into())), start_pos)); } - (b':', ..) => return Some((Token::Colon, start_pos)), + (':', ..) => return Some((Token::Colon, start_pos)), - (b'<', b'=') => { + ('<', '=') => { eat_next_and_advance(stream, pos); return Some((Token::LessThanEqualsTo, start_pos)); } - (b'<', b'-') => { + ('<', '-') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("<-".into())), start_pos)); } - (b'<', b'<') => { + ('<', '<') => { eat_next_and_advance(stream, pos); return Some(( @@ -2070,17 +2053,17 @@ fn get_next_token_inner( start_pos, )); } - (b'<', b'|') => { + ('<', '|') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("<|".into())), start_pos)); } - (b'<', ..) => return Some((Token::LessThan, start_pos)), + ('<', ..) => return Some((Token::LessThan, start_pos)), - (b'>', b'=') => { + ('>', '=') => { eat_next_and_advance(stream, pos); return Some((Token::GreaterThanEqualsTo, start_pos)); } - (b'>', b'>') => { + ('>', '>') => { eat_next_and_advance(stream, pos); return Some(( @@ -2093,9 +2076,9 @@ fn get_next_token_inner( start_pos, )); } - (b'>', ..) => return Some((Token::GreaterThan, start_pos)), + ('>', ..) => return Some((Token::GreaterThan, start_pos)), - (b'!', b'i') => { + ('!', 'i') => { stream.get_next().unwrap(); if stream.peek_next() == Some('n') { stream.get_next().unwrap(); @@ -2116,7 +2099,7 @@ fn get_next_token_inner( stream.unget('i'); return Some((Token::Bang, start_pos)); } - (b'!', b'=') => { + ('!', '=') => { eat_next_and_advance(stream, pos); if stream.peek_next() == Some('=') { @@ -2126,55 +2109,55 @@ fn get_next_token_inner( return Some((Token::NotEqualsTo, start_pos)); } - (b'!', b'.') => { + ('!', '.') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("!.".into())), start_pos)); } - (b'!', ..) => return Some((Token::Bang, start_pos)), + ('!', ..) => return Some((Token::Bang, start_pos)), - (b'|', b'|') => { + ('|', '|') => { eat_next_and_advance(stream, pos); return Some((Token::Or, start_pos)); } - (b'|', b'=') => { + ('|', '=') => { eat_next_and_advance(stream, pos); return Some((Token::OrAssign, start_pos)); } - (b'|', b'>') => { + ('|', '>') => { eat_next_and_advance(stream, pos); return Some((Token::Reserved(Box::new("|>".into())), start_pos)); } - (b'|', ..) => return Some((Token::Pipe, start_pos)), + ('|', ..) => return Some((Token::Pipe, start_pos)), - (b'&', b'&') => { + ('&', '&') => { eat_next_and_advance(stream, pos); return Some((Token::And, start_pos)); } - (b'&', b'=') => { + ('&', '=') => { eat_next_and_advance(stream, pos); return Some((Token::AndAssign, start_pos)); } - (b'&', ..) => return Some((Token::Ampersand, start_pos)), + ('&', ..) => return Some((Token::Ampersand, start_pos)), - (b'^', b'=') => { + ('^', '=') => { eat_next_and_advance(stream, pos); return Some((Token::XOrAssign, start_pos)); } - (b'^', ..) => return Some((Token::XOr, start_pos)), + ('^', ..) => return Some((Token::XOr, start_pos)), - (b'~', ..) => return Some((Token::Reserved(Box::new("~".into())), start_pos)), + ('~', ..) => return Some((Token::Reserved(Box::new("~".into())), start_pos)), - (b'%', b'=') => { + ('%', '=') => { eat_next_and_advance(stream, pos); return Some((Token::ModuloAssign, start_pos)); } - (b'%', ..) => return Some((Token::Modulo, start_pos)), + ('%', ..) => return Some((Token::Modulo, start_pos)), - (b'@', ..) => return Some((Token::Reserved(Box::new("@".into())), start_pos)), + ('@', ..) => return Some((Token::Reserved(Box::new("@".into())), start_pos)), - (b'$', ..) => return Some((Token::Reserved(Box::new("$".into())), start_pos)), + ('$', ..) => return Some((Token::Reserved(Box::new("$".into())), start_pos)), - (b'?', b'.') => { + ('?', '.') => { eat_next_and_advance(stream, pos); return Some(( #[cfg(not(feature = "no_object"))] @@ -2184,11 +2167,11 @@ fn get_next_token_inner( start_pos, )); } - (b'?', b'?') => { + ('?', '?') => { eat_next_and_advance(stream, pos); return Some((Token::DoubleQuestion, start_pos)); } - (b'?', b'[') => { + ('?', '[') => { eat_next_and_advance(stream, pos); return Some(( #[cfg(not(feature = "no_index"))] @@ -2198,7 +2181,12 @@ fn get_next_token_inner( start_pos, )); } - (b'?', ..) => return Some((Token::Reserved(Box::new("?".into())), start_pos)), + ('?', ..) => return Some((Token::Reserved(Box::new("?".into())), start_pos)), + + // letter or underscore ... + _ if is_id_first_alphabetic(c) || c == '_' => { + return Some(parse_identifier_token(stream, state, pos, start_pos, c)); + } _ if c.is_whitespace() => (), @@ -2323,6 +2311,8 @@ pub fn is_id_continue(x: char) -> bool { #[inline] #[must_use] pub fn is_reserved_keyword_or_symbol(syntax: &str) -> (bool, bool, bool) { + // This implementation is based upon a pre-calculated table generated + // by GNU gperf on the list of keywords. let utf8 = syntax.as_bytes(); let len = utf8.len(); let rounds = len.min(3); @@ -2342,7 +2332,13 @@ pub fn is_reserved_keyword_or_symbol(syntax: &str) -> (bool, bool, bool) { match RESERVED_LIST[hash_val] { ("", ..) => (false, false, false), - (s, true, a, b) => (s == syntax, a, b), + (s, true, a, b) => ( + // Fail early to avoid calling memcmp() + // Since we are already working with bytes, mind as well check the first one + s.len() == len && s.as_bytes()[0] == utf8[0] && s == syntax, + a, + b, + ), _ => (false, false, false), } } From 9ce581f745c214ddde241628ca846fa5fb183852 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 16 Mar 2023 13:30:32 +0800 Subject: [PATCH 39/41] Make track_progress inline. --- src/eval/data_check.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/eval/data_check.rs b/src/eval/data_check.rs index 563881ec..1bcbefe8 100644 --- a/src/eval/data_check.rs +++ b/src/eval/data_check.rs @@ -191,6 +191,7 @@ impl Engine { } /// Check if the number of operations stay within limit. + #[inline(always)] pub(crate) fn track_operation( &self, global: &mut GlobalRuntimeState, @@ -199,20 +200,16 @@ impl Engine { global.num_operations += 1; // Guard against too many operations - let max = self.max_operations(); - - if max > 0 && global.num_operations > max { - return Err(ERR::ErrorTooManyOperations(pos).into()); - } - - // Report progress - if let Some(ref progress) = self.progress { - match progress(global.num_operations) { - None => Ok(()), - Some(token) => Err(ERR::ErrorTerminated(token, pos).into()), - } + if self.max_operations() > 0 && global.num_operations > self.max_operations() { + Err(ERR::ErrorTooManyOperations(pos).into()) } else { - Ok(()) + self.progress + .as_ref() + .and_then(|progress| { + progress(global.num_operations) + .map(|token| Err(ERR::ErrorTerminated(token, pos).into())) + }) + .unwrap_or(Ok(())) } } } From c7d40945ee91bc90551fd09b5d363e88522342b6 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 16 Mar 2023 18:03:54 +0800 Subject: [PATCH 40/41] Attempt to prevent building lookup tables on dispatch. --- src/engine.rs | 6 ++ src/eval/expr.rs | 69 +++++++------ src/eval/stmt.rs | 258 ++++++++++++++++++++++++----------------------- 3 files changed, 180 insertions(+), 153 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index 1e35d21f..8ad61089 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -367,4 +367,10 @@ impl Engine { pub(crate) const fn is_debugger_registered(&self) -> bool { self.debugger_interface.is_some() } + + /// Imitation of std::hints::black_box which requires nightly. + #[inline(never)] + pub(crate) fn black_box() -> usize { + unsafe { core::ptr::read_volatile(&0_usize as *const usize) } + } } diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 0567e39e..91d944cf 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -239,17 +239,17 @@ impl Engine { // Coded this way for better branch prediction. // Popular branches are lifted out of the `match` statement into their own branches. + #[cfg(feature = "debugging")] + let reset = + self.run_debugger_with_reset(global, caches, scope, this_ptr.as_deref_mut(), expr)?; + #[cfg(feature = "debugging")] + auto_restore!(global if Some(reset) => move |g| g.debugger_mut().reset_status(reset)); + + self.track_operation(global, expr.position())?; + // Function calls should account for a relatively larger portion of expressions because // binary operators are also function calls. if let Expr::FnCall(x, pos) = expr { - #[cfg(feature = "debugging")] - let reset = - self.run_debugger_with_reset(global, caches, scope, this_ptr.as_deref_mut(), expr)?; - #[cfg(feature = "debugging")] - auto_restore!(global if Some(reset) => move |g| g.debugger_mut().reset_status(reset)); - - self.track_operation(global, expr.position())?; - return self.eval_fn_call_expr(global, caches, scope, this_ptr, x, *pos); } @@ -257,11 +257,6 @@ impl Engine { // We shouldn't do this for too many variants because, soon or later, the added comparisons // will cost more than the mis-predicted `match` branch. if let Expr::Variable(x, index, var_pos) = expr { - #[cfg(feature = "debugging")] - self.run_debugger(global, caches, scope, this_ptr.as_deref_mut(), expr)?; - - self.track_operation(global, expr.position())?; - return if index.is_none() && x.0.is_none() && x.3 == KEYWORD_THIS { this_ptr .ok_or_else(|| ERR::ErrorUnboundThis(*var_pos).into()) @@ -272,25 +267,41 @@ impl Engine { }; } - #[cfg(feature = "debugging")] - let reset = - self.run_debugger_with_reset(global, caches, scope, this_ptr.as_deref_mut(), expr)?; - #[cfg(feature = "debugging")] - auto_restore!(global if Some(reset) => move |g| g.debugger_mut().reset_status(reset)); + // Stop merging branches here! + Self::black_box(); - self.track_operation(global, expr.position())?; + // Constants + if let Expr::IntegerConstant(x, ..) = expr { + return Ok((*x).into()); + } + if let Expr::StringConstant(x, ..) = expr { + return Ok(x.clone().into()); + } + if let Expr::BoolConstant(x, ..) = expr { + return Ok((*x).into()); + } + + // Stop merging branches here! + Self::black_box(); + + #[cfg(not(feature = "no_float"))] + if let Expr::FloatConstant(x, ..) = expr { + return Ok((*x).into()); + } + if let Expr::CharConstant(x, ..) = expr { + return Ok((*x).into()); + } + if let Expr::Unit(..) = expr { + return Ok(Dynamic::UNIT); + } + if let Expr::DynamicConstant(x, ..) = expr { + return Ok(x.as_ref().clone()); + } + + // Stop merging branches here! + Self::black_box(); match expr { - // Constants - Expr::DynamicConstant(x, ..) => Ok(x.as_ref().clone()), - Expr::IntegerConstant(x, ..) => Ok((*x).into()), - #[cfg(not(feature = "no_float"))] - Expr::FloatConstant(x, ..) => Ok((*x).into()), - Expr::StringConstant(x, ..) => Ok(x.clone().into()), - Expr::CharConstant(x, ..) => Ok((*x).into()), - Expr::BoolConstant(x, ..) => Ok((*x).into()), - Expr::Unit(..) => Ok(Dynamic::UNIT), - // `... ${...} ...` Expr::InterpolatedString(x, _) => { let mut concat = SmartString::new_const(); diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 4846c9a7..4b863701 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -273,13 +273,14 @@ impl Engine { #[cfg(feature = "debugging")] auto_restore!(global if Some(reset) => move |g| g.debugger_mut().reset_status(reset)); + self.track_operation(global, stmt.position())?; + // Coded this way for better branch prediction. // Popular branches are lifted out of the `match` statement into their own branches. + // Hopefully the compiler won't undo all this work! // Function calls should account for a relatively larger portion of statements. if let Stmt::FnCall(x, pos) = stmt { - self.track_operation(global, stmt.position())?; - return self.eval_fn_call_expr(global, caches, scope, this_ptr, x, *pos); } @@ -289,8 +290,6 @@ impl Engine { if let Stmt::Assignment(x, ..) = stmt { let (op_info, BinaryExpr { lhs, rhs }) = &**x; - self.track_operation(global, stmt.position())?; - if let Expr::Variable(x, ..) = lhs { let rhs_val = self .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), rhs)? @@ -354,47 +353,146 @@ impl Engine { } } - self.track_operation(global, stmt.position())?; + // Stop merging branches here! + Self::black_box(); - match stmt { - // No-op - Stmt::Noop(..) => Ok(Dynamic::UNIT), - - // Expression as statement - Stmt::Expr(expr) => self - .eval_expr(global, caches, scope, this_ptr, expr) - .map(Dynamic::flatten), - - // Block scope - Stmt::Block(statements, ..) if statements.is_empty() => Ok(Dynamic::UNIT), - Stmt::Block(statements, ..) => { + // Block scope + if let Stmt::Block(statements, ..) = stmt { + return if statements.is_empty() { + Ok(Dynamic::UNIT) + } else { self.eval_stmt_block(global, caches, scope, this_ptr, statements, true) + }; + } + + if let Stmt::Var(x, options, pos) = stmt { + if !self.allow_shadowing() && scope.contains(&x.0) { + return Err(ERR::ErrorVariableExists(x.0.to_string(), *pos).into()); } + // Let/const statement + let (var_name, expr, index) = &**x; - // If statement - Stmt::If(x, ..) => { - let FlowControl { - expr, - body: if_block, - branch: else_block, - } = &**x; + let access = if options.contains(ASTFlags::CONSTANT) { + AccessMode::ReadOnly + } else { + AccessMode::ReadWrite + }; + let export = options.contains(ASTFlags::EXPORTED); - let guard_val = self - .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), expr)? - .as_bool() - .map_err(|typ| self.make_type_mismatch_err::(typ, expr.position()))?; + // Check variable definition filter + if let Some(ref filter) = self.def_var_filter { + let will_shadow = scope.contains(var_name); + let is_const = access == AccessMode::ReadOnly; + let info = VarDefInfo { + name: var_name, + is_const, + nesting_level: global.scope_level, + will_shadow, + }; + let orig_scope_len = scope.len(); + let context = + EvalContext::new(self, global, caches, scope, this_ptr.as_deref_mut()); + let filter_result = filter(true, info, context); - match guard_val { - true if !if_block.is_empty() => { - self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) - } - false if !else_block.is_empty() => { - self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) - } - _ => Ok(Dynamic::UNIT), + if orig_scope_len != scope.len() { + // The scope is changed, always search from now on + global.always_search_scope = true; + } + + if !filter_result? { + return Err(ERR::ErrorForbiddenVariable(var_name.to_string(), *pos).into()); } } + // Evaluate initial value + let mut value = self + .eval_expr(global, caches, scope, this_ptr, expr)? + .flatten() + .intern_string(self); + + let _alias = if !rewind_scope { + // Put global constants into global module + #[cfg(not(feature = "no_function"))] + #[cfg(not(feature = "no_module"))] + if global.scope_level == 0 + && access == AccessMode::ReadOnly + && global.lib.iter().any(|m| !m.is_empty()) + { + crate::func::locked_write(global.constants.get_or_insert_with(|| { + crate::Shared::new(crate::Locked::new(std::collections::BTreeMap::new())) + })) + .insert(var_name.name.clone(), value.clone()); + } + + if export { + Some(var_name) + } else { + None + } + } else if export { + unreachable!("exported variable not on global level"); + } else { + None + }; + + if let Some(index) = index { + value.set_access_mode(access); + *scope.get_mut_by_index(scope.len() - index.get()) = value; + } else { + scope.push_entry(var_name.name.clone(), access, value); + } + + #[cfg(not(feature = "no_module"))] + if let Some(alias) = _alias { + scope.add_alias_by_index(scope.len() - 1, alias.as_str().into()); + } + + return Ok(Dynamic::UNIT); + } + + // Stop merging branches here! + Self::black_box(); + + // If statement + if let Stmt::If(x, ..) = stmt { + let FlowControl { + expr, + body: if_block, + branch: else_block, + } = &**x; + + let guard_val = self + .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), expr)? + .as_bool() + .map_err(|typ| self.make_type_mismatch_err::(typ, expr.position()))?; + + return match guard_val { + true if !if_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) + } + false if !else_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) + } + _ => Ok(Dynamic::UNIT), + }; + } + + // Expression as statement + if let Stmt::Expr(expr) = stmt { + return self + .eval_expr(global, caches, scope, this_ptr, expr) + .map(Dynamic::flatten); + } + + // No-op + if let Stmt::Noop(..) = stmt { + return Ok(Dynamic::UNIT); + } + + // Stop merging branches here! + Self::black_box(); + + match stmt { // Switch statement Stmt::Switch(x, ..) => { let ( @@ -766,94 +864,6 @@ impl Engine { // Empty return Stmt::Return(None, .., pos) => Err(ERR::Return(Dynamic::UNIT, *pos).into()), - // Let/const statement - shadowing disallowed - Stmt::Var(x, .., pos) if !self.allow_shadowing() && scope.contains(&x.0) => { - Err(ERR::ErrorVariableExists(x.0.to_string(), *pos).into()) - } - // Let/const statement - Stmt::Var(x, options, pos) => { - let (var_name, expr, index) = &**x; - - let access = if options.contains(ASTFlags::CONSTANT) { - AccessMode::ReadOnly - } else { - AccessMode::ReadWrite - }; - let export = options.contains(ASTFlags::EXPORTED); - - // Check variable definition filter - if let Some(ref filter) = self.def_var_filter { - let will_shadow = scope.contains(var_name); - let is_const = access == AccessMode::ReadOnly; - let info = VarDefInfo { - name: var_name, - is_const, - nesting_level: global.scope_level, - will_shadow, - }; - let orig_scope_len = scope.len(); - let context = - EvalContext::new(self, global, caches, scope, this_ptr.as_deref_mut()); - let filter_result = filter(true, info, context); - - if orig_scope_len != scope.len() { - // The scope is changed, always search from now on - global.always_search_scope = true; - } - - if !filter_result? { - return Err(ERR::ErrorForbiddenVariable(var_name.to_string(), *pos).into()); - } - } - - // Evaluate initial value - let mut value = self - .eval_expr(global, caches, scope, this_ptr, expr)? - .flatten() - .intern_string(self); - - let _alias = if !rewind_scope { - // Put global constants into global module - #[cfg(not(feature = "no_function"))] - #[cfg(not(feature = "no_module"))] - if global.scope_level == 0 - && access == AccessMode::ReadOnly - && global.lib.iter().any(|m| !m.is_empty()) - { - crate::func::locked_write(global.constants.get_or_insert_with(|| { - crate::Shared::new( - crate::Locked::new(std::collections::BTreeMap::new()), - ) - })) - .insert(var_name.name.clone(), value.clone()); - } - - if export { - Some(var_name) - } else { - None - } - } else if export { - unreachable!("exported variable not on global level"); - } else { - None - }; - - if let Some(index) = index { - value.set_access_mode(access); - *scope.get_mut_by_index(scope.len() - index.get()) = value; - } else { - scope.push_entry(var_name.name.clone(), access, value); - } - - #[cfg(not(feature = "no_module"))] - if let Some(alias) = _alias { - scope.add_alias_by_index(scope.len() - 1, alias.as_str().into()); - } - - Ok(Dynamic::UNIT) - } - // Import statement #[cfg(not(feature = "no_module"))] Stmt::Import(x, _pos) => { From a3324ceb3d916e965c6286cfe67ca894fc8ab4a1 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Thu, 16 Mar 2023 19:00:59 +0800 Subject: [PATCH 41/41] Reduce lifted out variants. --- src/engine.rs | 7 ++++ src/eval/expr.rs | 51 ++++++++++--------------- src/eval/stmt.rs | 99 ++++++++++++++++++++++-------------------------- 3 files changed, 72 insertions(+), 85 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index 8ad61089..e4068fc7 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -369,8 +369,15 @@ impl Engine { } /// Imitation of std::hints::black_box which requires nightly. + #[cfg(not(target_family = "wasm"))] #[inline(never)] pub(crate) fn black_box() -> usize { unsafe { core::ptr::read_volatile(&0_usize as *const usize) } } + /// Imitation of std::hints::black_box which requires nightly. + #[cfg(target_family = "wasm")] + #[inline(always)] + pub(crate) fn black_box() -> usize { + 0 + } } diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 91d944cf..c2aaf79d 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -254,8 +254,6 @@ impl Engine { } // Then variable access. - // We shouldn't do this for too many variants because, soon or later, the added comparisons - // will cost more than the mis-predicted `match` branch. if let Expr::Variable(x, index, var_pos) = expr { return if index.is_none() && x.0.is_none() && x.3 == KEYWORD_THIS { this_ptr @@ -267,41 +265,27 @@ impl Engine { }; } - // Stop merging branches here! - Self::black_box(); - - // Constants + // Then integer constants. if let Expr::IntegerConstant(x, ..) = expr { return Ok((*x).into()); } - if let Expr::StringConstant(x, ..) = expr { - return Ok(x.clone().into()); - } - if let Expr::BoolConstant(x, ..) = expr { - return Ok((*x).into()); - } - - // Stop merging branches here! - Self::black_box(); - - #[cfg(not(feature = "no_float"))] - if let Expr::FloatConstant(x, ..) = expr { - return Ok((*x).into()); - } - if let Expr::CharConstant(x, ..) = expr { - return Ok((*x).into()); - } - if let Expr::Unit(..) = expr { - return Ok(Dynamic::UNIT); - } - if let Expr::DynamicConstant(x, ..) = expr { - return Ok(x.as_ref().clone()); - } // Stop merging branches here! + // We shouldn't lift out too many variants because, soon or later, the added comparisons + // will cost more than the mis-predicted `match` branch. Self::black_box(); match expr { + // Constants + Expr::IntegerConstant(..) => unreachable!(), + Expr::StringConstant(x, ..) => Ok(x.clone().into()), + Expr::BoolConstant(x, ..) => Ok((*x).into()), + #[cfg(not(feature = "no_float"))] + Expr::FloatConstant(x, ..) => Ok((*x).into()), + Expr::CharConstant(x, ..) => Ok((*x).into()), + Expr::Unit(..) => Ok(Dynamic::UNIT), + Expr::DynamicConstant(x, ..) => Ok(x.as_ref().clone()), + // `... ${...} ...` Expr::InterpolatedString(x, _) => { let mut concat = SmartString::new_const(); @@ -445,8 +429,13 @@ impl Engine { .and_then(|r| self.check_data_size(r, expr.start_position())) } - Expr::Stmt(x) if x.is_empty() => Ok(Dynamic::UNIT), - Expr::Stmt(x) => self.eval_stmt_block(global, caches, scope, this_ptr, x, true), + Expr::Stmt(x) => { + if x.is_empty() { + Ok(Dynamic::UNIT) + } else { + self.eval_stmt_block(global, caches, scope, this_ptr, x, true) + } + } #[cfg(not(feature = "no_index"))] Expr::Index(..) => { diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 4b863701..1f4ae51d 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -277,7 +277,6 @@ impl Engine { // Coded this way for better branch prediction. // Popular branches are lifted out of the `match` statement into their own branches. - // Hopefully the compiler won't undo all this work! // Function calls should account for a relatively larger portion of statements. if let Stmt::FnCall(x, pos) = stmt { @@ -285,8 +284,6 @@ impl Engine { } // Then assignments. - // We shouldn't do this for too many variants because, soon or later, the added comparisons - // will cost more than the mis-predicted `match` branch. if let Stmt::Assignment(x, ..) = stmt { let (op_info, BinaryExpr { lhs, rhs }) = &**x; @@ -353,22 +350,12 @@ impl Engine { } } - // Stop merging branches here! - Self::black_box(); - - // Block scope - if let Stmt::Block(statements, ..) = stmt { - return if statements.is_empty() { - Ok(Dynamic::UNIT) - } else { - self.eval_stmt_block(global, caches, scope, this_ptr, statements, true) - }; - } - + // Then variable definitions. if let Stmt::Var(x, options, pos) = stmt { if !self.allow_shadowing() && scope.contains(&x.0) { return Err(ERR::ErrorVariableExists(x.0.to_string(), *pos).into()); } + // Let/const statement let (var_name, expr, index) = &**x; @@ -451,48 +438,52 @@ impl Engine { } // Stop merging branches here! - Self::black_box(); - - // If statement - if let Stmt::If(x, ..) = stmt { - let FlowControl { - expr, - body: if_block, - branch: else_block, - } = &**x; - - let guard_val = self - .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), expr)? - .as_bool() - .map_err(|typ| self.make_type_mismatch_err::(typ, expr.position()))?; - - return match guard_val { - true if !if_block.is_empty() => { - self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) - } - false if !else_block.is_empty() => { - self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) - } - _ => Ok(Dynamic::UNIT), - }; - } - - // Expression as statement - if let Stmt::Expr(expr) = stmt { - return self - .eval_expr(global, caches, scope, this_ptr, expr) - .map(Dynamic::flatten); - } - - // No-op - if let Stmt::Noop(..) = stmt { - return Ok(Dynamic::UNIT); - } - - // Stop merging branches here! + // We shouldn't lift out too many variants because, soon or later, the added comparisons + // will cost more than the mis-predicted `match` branch. Self::black_box(); match stmt { + // No-op + Stmt::Noop(..) => Ok(Dynamic::UNIT), + + // Expression as statement + Stmt::Expr(expr) => self + .eval_expr(global, caches, scope, this_ptr, expr) + .map(Dynamic::flatten), + + // Block scope + Stmt::Block(statements, ..) => { + if statements.is_empty() { + Ok(Dynamic::UNIT) + } else { + self.eval_stmt_block(global, caches, scope, this_ptr, statements, true) + } + } + + // If statement + Stmt::If(x, ..) => { + let FlowControl { + expr, + body: if_block, + branch: else_block, + } = &**x; + + let guard_val = self + .eval_expr(global, caches, scope, this_ptr.as_deref_mut(), expr)? + .as_bool() + .map_err(|typ| self.make_type_mismatch_err::(typ, expr.position()))?; + + match guard_val { + true if !if_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, if_block, true) + } + false if !else_block.is_empty() => { + self.eval_stmt_block(global, caches, scope, this_ptr, else_block, true) + } + _ => Ok(Dynamic::UNIT), + } + } + // Switch statement Stmt::Switch(x, ..) => { let (