More code refactor.

CHANGELOG.md

@@ -14,6 +14,12 @@ Breaking API changes
 
 * The callback for initializing a debugger instance has changed to `Fn(&Engine, Debugger) -> Debugger`. This allows more control over the initial setup of the debugger.
 
+Deprecated API's
+----------------
+
+* `Module::with_capacity` is deprecated.
+* The internal method `Engine::eval_statements_raw` is deprecated.
+
 Net features
 ------------
 

src/api/build_type.rs

@@ -101,7 +101,7 @@ impl<'a, T: Variant + Clone> TypeBuilder<'a, T> {
         Self {
             engine,
             name: None,
-            _marker: PhantomData::default(),
+            _marker: PhantomData,
         }
     }
 }

src/api/call_fn.rs

@@ -81,7 +81,7 @@ impl Engine {
     ///
     /// The [`AST`] is evaluated before calling the function.
     /// This allows a script to load the necessary modules.
-    /// This is usually desired. If not, use [`call_fn_with_options`] instead.
+    /// This is usually desired. If not, use [`call_fn_with_options`][Engine::call_fn_with_options] instead.
     ///
     /// # Example
     ///
@@ -276,7 +276,7 @@ impl Engine {
         });
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             global.debugger_mut().status = crate::eval::DebuggerStatus::Terminate;
             let node = &crate::ast::Stmt::Noop(Position::NONE);
             self.run_debugger(global, caches, scope, this_ptr, node)?;

src/api/compile.rs

@@ -218,13 +218,10 @@ impl Engine {
         scripts: impl AsRef<[S]>,
         optimization_level: OptimizationLevel,
     ) -> ParseResult<AST> {
-        let (stream, tokenizer_control) = self.lex_raw(
+        let (stream, tc) = self.lex_raw(scripts.as_ref(), self.token_mapper.as_deref());
-            scripts.as_ref(),
-            self.token_mapper.as_ref().map(<_>::as_ref),
-        );
         let interned_strings = &mut *locked_write(&self.interned_strings);
-        let mut state = ParseState::new(scope, interned_strings, tokenizer_control);
+        let state = &mut ParseState::new(scope, interned_strings, tc);
-        let mut _ast = self.parse(&mut stream.peekable(), &mut state, optimization_level)?;
+        let mut _ast = self.parse(stream.peekable(), state, optimization_level)?;
         #[cfg(feature = "metadata")]
         _ast.set_doc(state.tokenizer_control.borrow().global_comments.join("\n"));
         Ok(_ast)
@@ -292,12 +289,9 @@ impl Engine {
         script: impl AsRef<str>,
     ) -> ParseResult<AST> {
         let scripts = [script];
-        let (stream, tokenizer_control) =
+        let (stream, t) = self.lex_raw(&scripts, self.token_mapper.as_deref());
-            self.lex_raw(&scripts, self.token_mapper.as_ref().map(<_>::as_ref));
-
-        let mut peekable = stream.peekable();
         let interned_strings = &mut *locked_write(&self.interned_strings);
-        let mut state = ParseState::new(scope, interned_strings, tokenizer_control);
+        let state = &mut ParseState::new(scope, interned_strings, t);
-        self.parse_global_expr(&mut peekable, &mut state, |_| {}, self.optimization_level)
+        self.parse_global_expr(stream.peekable(), state, |_| {}, self.optimization_level)
     }
 }

src/api/custom_syntax.rs

@@ -259,7 +259,7 @@ impl Engine {
                     // Make it a custom keyword/symbol if it is disabled or reserved
                     if (self
                         .disabled_symbols
-                        .as_ref()
+                        .as_deref()
                         .map_or(false, |m| m.contains(s))
                         || token.map_or(false, |v| v.is_reserved()))
                         && !self

src/api/deprecated.rs

@@ -26,7 +26,7 @@ impl Engine {
     ///
     /// This method is deprecated. Use [`run_file`][Engine::run_file] instead.
     ///
-    /// This method will be removed in the next majocd cr version.
+    /// This method will be removed in the next major version.
     #[deprecated(since = "1.1.0", note = "use `run_file` instead")]
     #[cfg(not(feature = "no_std"))]
     #[cfg(not(target_family = "wasm"))]
@@ -137,12 +137,6 @@ impl Engine {
     }
     /// Call a script function defined in an [`AST`] with multiple [`Dynamic`] arguments.
     ///
-    /// The following options are available:
-    ///
-    /// * whether to evaluate the [`AST`] to load necessary modules before calling the function
-    /// * whether to rewind the [`Scope`] after the function call
-    /// * a value for binding to the `this` pointer (if any)
-    ///
     /// Not available under `no_function`.
     ///
     /// # Deprecated
@@ -253,12 +247,6 @@ impl Engine {
     /// This method is deprecated. Use [`register_indexer_get`][Engine::register_indexer_get] instead.
     ///
     /// This method will be removed in the next major version.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the type is [`Array`][crate::Array], [`Map`][crate::Map], [`String`],
-    /// [`ImmutableString`][crate::ImmutableString], `&str` or [`INT`][crate::INT].
-    /// Indexers for arrays, object maps, strings and integers cannot be registered.
     #[deprecated(since = "1.9.1", note = "use `register_indexer_get` instead")]
     #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]
     #[inline(always)]
@@ -284,12 +272,6 @@ impl Engine {
     /// This method is deprecated. Use [`register_indexer_set`][Engine::register_indexer_set] instead.
     ///
     /// This method will be removed in the next major version.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the type is [`Array`][crate::Array], [`Map`][crate::Map], [`String`],
-    /// [`ImmutableString`][crate::ImmutableString], `&str` or [`INT`][crate::INT].
-    /// Indexers for arrays, object maps, strings and integers cannot be registered.
     #[deprecated(since = "1.9.1", note = "use `register_indexer_set` instead")]
     #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]
     #[inline(always)]
@@ -316,34 +298,6 @@ impl Engine {
     /// Use [`register_custom_syntax_with_state_raw`][Engine::register_custom_syntax_with_state_raw] instead.
     ///
     /// This method will be removed in the next major version.
-    ///
-    /// # WARNING - Low Level API
-    ///
-    /// This function is very low level.
-    ///
-    /// * `scope_may_be_changed` specifies variables have been added/removed by this custom syntax.
-    /// * `parse` is the parsing function.
-    /// * `func` is the implementation function.
-    ///
-    /// All custom keywords used as symbols must be manually registered via [`Engine::register_custom_operator`].
-    /// Otherwise, they won't be recognized.
-    ///
-    /// # Parsing Function Signature
-    ///
-    /// The parsing function has the following signature:
-    ///
-    /// `Fn(symbols: &[ImmutableString], look_ahead: &str) -> Result<Option<ImmutableString>, ParseError>`
-    ///
-    /// where:
-    /// * `symbols`: a slice of symbols that have been parsed so far, possibly containing `$expr$` and/or `$block$`;
-    ///   `$ident$` and other literal markers are replaced by the actual text
-    /// * `look_ahead`: a string slice containing the next symbol that is about to be read
-    ///
-    /// ## Return value
-    ///
-    /// * `Ok(None)`: parsing complete and there are no more symbols to match.
-    /// * `Ok(Some(symbol))`: the next symbol to match, which can also be `$expr$`, `$ident$` or `$block$`.
-    /// * `Err(ParseError)`: error that is reflected back to the [`Engine`], normally `ParseError(ParseErrorType::BadInput(LexError::ImproperSymbol(message)), Position::NONE)` to indicate a syntax error, but it can be any [`ParseError`][crate::ParseError].
     #[deprecated(
         since = "1.11.0",
         note = "use `register_custom_syntax_with_state_raw` instead"
@@ -368,6 +322,24 @@ impl Engine {
             move |context, expressions, _| func(context, expressions),
         )
     }
+    /// _(internals)_ Evaluate a list of statements with no `this` pointer.
+    /// Exported under the `internals` feature only.
+    ///
+    /// # Deprecated
+    ///
+    /// This method is deprecated. It will be removed in the next major version.
+    #[cfg(feature = "internals")]
+    #[inline(always)]
+    #[deprecated(since = "1.12.0")]
+    pub fn eval_statements_raw(
+        &self,
+        global: &mut crate::eval::GlobalRuntimeState,
+        caches: &mut crate::eval::Caches,
+        scope: &mut Scope,
+        statements: &[crate::ast::Stmt],
+    ) -> RhaiResult {
+        self.eval_global_statements(global, caches, scope, statements)
+    }
 }
 
 impl Dynamic {

src/api/eval.rs

@@ -121,15 +121,14 @@ impl Engine {
         let ast = {
             let interned_strings = &mut *locked_write(&self.interned_strings);
 
-            let (stream, tokenizer_control) =
+            let (stream, tc) = self.lex_raw(&scripts, self.token_mapper.as_deref());
-                self.lex_raw(&scripts, self.token_mapper.as_ref().map(<_>::as_ref));
 
-            let mut state = ParseState::new(scope, interned_strings, tokenizer_control);
+            let state = &mut ParseState::new(scope, interned_strings, tc);
 
             // No need to optimize a lone expression
             self.parse_global_expr(
-                &mut stream.peekable(),
+                stream.peekable(),
-                &mut state,
+                state,
                 |_| {},
                 #[cfg(not(feature = "no_optimize"))]
                 OptimizationLevel::None,
@@ -243,7 +242,7 @@ impl Engine {
         let result = self.eval_global_statements(global, caches, scope, statements);
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             global.debugger_mut().status = crate::eval::DebuggerStatus::Terminate;
             let mut this = Dynamic::NULL;
             let node = &crate::ast::Stmt::Noop(Position::NONE);
@@ -263,25 +262,6 @@ impl Engine {
 
         result
     }
-    /// _(internals)_ Evaluate a list of statements with no `this` pointer.
-    /// Exported under the `internals` feature only.
-    ///
-    /// This is commonly used to evaluate a list of statements in an [`AST`] or a script function body.
-    ///
-    /// # WARNING - Low Level API
-    ///
-    /// This function is very low level.
-    #[cfg(feature = "internals")]
-    #[inline(always)]
-    pub fn eval_statements_raw(
-        &self,
-        global: &mut GlobalRuntimeState,
-        caches: &mut Caches,
-        scope: &mut Scope,
-        statements: &[crate::ast::Stmt],
-    ) -> RhaiResult {
-        self.eval_global_statements(global, caches, scope, statements)
-    }
 }
 
 /// Evaluate a string as a script, returning the result value or an error.

src/api/events.rs

@@ -362,7 +362,7 @@ impl Engine {
             + SendSync
             + 'static,
     ) -> &mut Self {
-        self.debugger = Some(Box::new((Box::new(init), Box::new(callback))));
+        self.debugger_interface = Some(Box::new((Box::new(init), Box::new(callback))));
         self
     }
 }

src/api/json.rs

@@ -120,11 +120,11 @@ impl Engine {
         let ast = {
             let scope = Scope::new();
             let interned_strings = &mut *locked_write(&self.interned_strings);
-            let mut state = ParseState::new(&scope, interned_strings, tokenizer_control);
+            let state = &mut ParseState::new(&scope, interned_strings, tokenizer_control);
 
             self.parse_global_expr(
-                &mut stream.peekable(),
+                stream.peekable(),
-                &mut state,
+                state,
                 |s| s.flags |= ParseSettingFlags::DISALLOW_UNQUOTED_MAP_PROPERTIES,
                 #[cfg(not(feature = "no_optimize"))]
                 OptimizationLevel::None,

src/api/mod.rs

@@ -1,7 +1,5 @@
 //! Module defining the public API of the Rhai engine.
 
-pub mod type_names;
-
 pub mod eval;
 
 pub mod run;
@@ -24,15 +22,6 @@ pub mod limits;
 
 pub mod events;
 
-pub mod custom_syntax;
-
-pub mod deprecated;
-
-pub mod build_type;
-
-#[cfg(feature = "metadata")]
-pub mod definitions;
-
 use crate::{Dynamic, Engine, Identifier};
 
 #[cfg(feature = "no_std")]
@@ -291,3 +280,14 @@ impl Engine {
         0
     }
 }
+
+pub mod type_names;
+
+pub mod custom_syntax;
+
+pub mod build_type;
+
+#[cfg(feature = "metadata")]
+pub mod definitions;
+
+pub mod deprecated;

src/api/optimize.rs

@@ -51,20 +51,14 @@ impl Engine {
     ) -> AST {
         let mut ast = ast;
 
-        #[cfg(not(feature = "no_function"))]
+        let mut _new_ast = self.optimize_into_ast(
-        let functions = ast
-            .shared_lib()
-            .iter_fn()
-            .filter(|f| f.func.is_script())
-            .map(|f| f.func.get_script_fn_def().unwrap().clone())
-            .collect();
-
-        let mut _new_ast = crate::optimizer::optimize_into_ast(
-            self,
             scope,
             ast.take_statements(),
             #[cfg(not(feature = "no_function"))]
-            functions,
+            ast.shared_lib()
+                .iter_fn()
+                .map(|f| f.func.get_script_fn_def().expect("`ScriptFnDef").clone())
+                .collect(),
             optimization_level,
         );
 

src/api/run.rs

@@ -58,14 +58,10 @@ impl Engine {
     pub fn run_with_scope(&self, scope: &mut Scope, script: &str) -> RhaiResultOf<()> {
         let scripts = [script];
         let ast = {
+            let (stream, tc) = self.lex_raw(&scripts, self.token_mapper.as_deref());
             let interned_strings = &mut *locked_write(&self.interned_strings);
-
+            let state = &mut ParseState::new(scope, interned_strings, tc);
-            let (stream, tokenizer_control) =
+            self.parse(stream.peekable(), state, self.optimization_level)?
-                self.lex_raw(&scripts, self.token_mapper.as_ref().map(<_>::as_ref));
-
-            let mut state = ParseState::new(scope, interned_strings, tokenizer_control);
-
-            self.parse(&mut stream.peekable(), &mut state, self.optimization_level)?
         };
         self.run_ast_with_scope(scope, &ast)
     }
@@ -136,7 +132,7 @@ impl Engine {
         }
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             global.debugger_mut().status = crate::eval::DebuggerStatus::Terminate;
             let mut this = crate::Dynamic::NULL;
             let node = &crate::ast::Stmt::Noop(crate::Position::NONE);

src/ast/expr.rs

@@ -208,7 +208,8 @@ pub struct FnCallExpr {
     /// Does this function call capture the parent scope?
     pub capture_parent_scope: bool,
     /// Is this function call a native operator?
-    pub op_token: Option<Token>,
+    /// Otherwise set to [`Token::NonToken`].
+    pub op_token: Token,
 }
 
 impl fmt::Debug for FnCallExpr {
@@ -223,8 +224,8 @@ impl fmt::Debug for FnCallExpr {
         ff.field("hash", &self.hashes)
             .field("name", &self.name)
             .field("args", &self.args);
-        if let Some(ref token) = self.op_token {
+        if self.op_token != Token::NonToken {
-            ff.field("op_token", token);
+            ff.field("op_token", &self.op_token);
         }
         if self.capture_parent_scope {
             ff.field("capture_parent_scope", &self.capture_parent_scope);
@@ -589,7 +590,7 @@ impl Expr {
                     hashes: calc_fn_hash(None, f.fn_name(), 1).into(),
                     args: once(Self::StringConstant(f.fn_name().into(), pos)).collect(),
                     capture_parent_scope: false,
-                    op_token: None,
+                    op_token: Token::NonToken,
                 }
                 .into(),
                 pos,

src/ast/script_fn.rs

@@ -24,7 +24,7 @@ pub struct EncapsulatedEnviron {
     /// Imported [modules][crate::Module].
     pub imports: Box<[(ImmutableString, crate::SharedModule)]>,
     /// Globally-defined constants.
-    pub constants: Option<crate::eval::GlobalConstants>,
+    pub constants: Option<crate::eval::SharedGlobalConstants>,
 }
 
 /// _(internals)_ A type containing information on a script-defined function.

src/ast/stmt.rs

@@ -61,10 +61,8 @@ impl OpAssignment {
     #[must_use]
     #[inline(always)]
     pub fn new_op_assignment(name: &str, pos: Position) -> Self {
-        Self::new_op_assignment_from_token(
+        let op = Token::lookup_symbol_from_syntax(name).expect("operator");
-            &Token::lookup_symbol_from_syntax(name).expect("operator"),
+        Self::new_op_assignment_from_token(op, pos)
-            pos,
-        )
     }
     /// Create a new [`OpAssignment`] from a [`Token`].
     ///
@@ -72,10 +70,11 @@ impl OpAssignment {
     ///
     /// Panics if the token is not an op-assignment operator.
     #[must_use]
-    pub fn new_op_assignment_from_token(op: &Token, pos: Position) -> Self {
+    pub fn new_op_assignment_from_token(op: Token, pos: Position) -> Self {
         let op_raw = op
             .get_base_op_from_assignment()
             .expect("op-assignment operator");
+
         Self {
             hash_op_assign: calc_fn_hash(None, op.literal_syntax(), 2),
             hash_op: calc_fn_hash(None, op_raw.literal_syntax(), 2),
@@ -92,10 +91,8 @@ impl OpAssignment {
     #[must_use]
     #[inline(always)]
     pub fn new_op_assignment_from_base(name: &str, pos: Position) -> Self {
-        Self::new_op_assignment_from_base_token(
+        let op = Token::lookup_symbol_from_syntax(name).expect("operator");
-            &Token::lookup_symbol_from_syntax(name).expect("operator"),
+        Self::new_op_assignment_from_base_token(op, pos)
-            pos,
-        )
     }
     /// Convert a [`Token`] into a new [`OpAssignment`].
     ///
@@ -104,8 +101,8 @@ impl OpAssignment {
     /// Panics if the token is cannot be converted into an op-assignment operator.
     #[inline(always)]
     #[must_use]
-    pub fn new_op_assignment_from_base_token(op: &Token, pos: Position) -> Self {
+    pub fn new_op_assignment_from_base_token(op: Token, pos: Position) -> Self {
-        Self::new_op_assignment_from_token(&op.convert_to_op_assignment().expect("operator"), pos)
+        Self::new_op_assignment_from_token(op.convert_to_op_assignment().expect("operator"), pos)
     }
 }
 

src/engine.rs

@@ -147,7 +147,7 @@ pub struct Engine {
 
     /// Callback closure for debugging.
     #[cfg(feature = "debugging")]
-    pub(crate) debugger: Option<
+    pub(crate) debugger_interface: Option<
         Box<(
             Box<crate::eval::OnDebuggingInit>,
             Box<crate::eval::OnDebuggerCallback>,
@@ -305,7 +305,7 @@ impl Engine {
             limits: crate::api::limits::Limits::new(),
 
             #[cfg(feature = "debugging")]
-            debugger: None,
+            debugger_interface: None,
         };
 
         // Add the global namespace module
@@ -348,4 +348,12 @@ impl Engine {
     pub fn const_empty_string(&self) -> ImmutableString {
         self.get_interned_string("")
     }
+
+    /// Is there a debugger interface registered with this [`Engine`]?
+    #[cfg(feature = "debugging")]
+    #[inline(always)]
+    #[must_use]
+    pub(crate) const fn is_debugger_registered(&self) -> bool {
+        self.debugger_interface.is_some()
+    }
 }

src/eval/chaining.rs

@@ -5,8 +5,8 @@ use super::{Caches, GlobalRuntimeState, Target};
 use crate::ast::{ASTFlags, Expr, OpAssignment};
 use crate::config::hashing::SusLock;
 use crate::engine::{FN_IDX_GET, FN_IDX_SET};
-use crate::types::dynamic::Union;
+use crate::tokenizer::Token;
-use crate::types::RestoreOnDrop;
+use crate::types::{dynamic::Union, RestoreOnDrop};
 use crate::{
     calc_fn_hash, Dynamic, Engine, FnArgsVec, Position, RhaiResult, RhaiResultOf, Scope, ERR,
 };
@@ -76,7 +76,7 @@ impl Engine {
             global,
             caches,
             FN_IDX_GET,
-            None,
+            Token::NonToken,
             hash_idx().0,
             &mut [target, idx],
             true,
@@ -105,7 +105,7 @@ impl Engine {
             global,
             caches,
             FN_IDX_SET,
-            None,
+            Token::NonToken,
             hash_idx().1,
             &mut [target, idx, new_val],
             is_ref_mut,
@@ -766,7 +766,14 @@ impl Engine {
                             let args = &mut [target.as_mut()];
                             let (mut orig_val, ..) = self
                                 .exec_native_fn_call(
-                                    global, caches, getter, None, *hash_get, args, is_ref_mut, *pos,
+                                    global,
+                                    caches,
+                                    getter,
+                                    Token::NonToken,
+                                    *hash_get,
+                                    args,
+                                    is_ref_mut,
+                                    *pos,
                                 )
                                 .or_else(|err| match *err {
                                     // Try an indexer if property does not exist
@@ -799,7 +806,14 @@ impl Engine {
 
                         let args = &mut [target.as_mut(), &mut new_val];
                         self.exec_native_fn_call(
-                            global, caches, setter, None, *hash_set, args, is_ref_mut, *pos,
+                            global,
+                            caches,
+                            setter,
+                            Token::NonToken,
+                            *hash_set,
+                            args,
+                            is_ref_mut,
+                            *pos,
                         )
                         .or_else(|err| match *err {
                             // Try an indexer if property does not exist
@@ -825,7 +839,14 @@ impl Engine {
                         let ((getter, hash_get), _, name) = &**x;
                         let args = &mut [target.as_mut()];
                         self.exec_native_fn_call(
-                            global, caches, getter, None, *hash_get, args, is_ref_mut, *pos,
+                            global,
+                            caches,
+                            getter,
+                            Token::NonToken,
+                            *hash_get,
+                            args,
+                            is_ref_mut,
+                            *pos,
                         )
                         .map_or_else(
                             |err| match *err {
@@ -921,7 +942,13 @@ impl Engine {
                                 // Assume getters are always pure
                                 let (mut val, ..) = self
                                     .exec_native_fn_call(
-                                        global, caches, getter, None, *hash_get, args, is_ref_mut,
+                                        global,
+                                        caches,
+                                        getter,
+                                        Token::NonToken,
+                                        *hash_get,
+                                        args,
+                                        is_ref_mut,
                                         pos,
                                     )
                                     .or_else(|err| match *err {
@@ -955,7 +982,13 @@ impl Engine {
                                     let mut arg_values = [target.as_mut(), val.as_mut()];
                                     let args = &mut arg_values;
                                     self.exec_native_fn_call(
-                                        global, caches, setter, None, *hash_set, args, is_ref_mut,
+                                        global,
+                                        caches,
+                                        setter,
+                                        Token::NonToken,
+                                        *hash_set,
+                                        args,
+                                        is_ref_mut,
                                         pos,
                                     )
                                     .or_else(

src/eval/debugger.rs

@@ -415,7 +415,7 @@ impl Engine {
         this_ptr: &mut Dynamic,
         node: impl Into<ASTNode<'a>>,
     ) -> RhaiResultOf<()> {
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             if let Some(cmd) =
                 self.run_debugger_with_reset_raw(global, caches, scope, this_ptr, node)?
             {
@@ -440,7 +440,7 @@ impl Engine {
         this_ptr: &mut Dynamic,
         node: impl Into<ASTNode<'a>>,
     ) -> RhaiResultOf<Option<DebuggerStatus>> {
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             self.run_debugger_with_reset_raw(global, caches, scope, this_ptr, node)
         } else {
             Ok(None)
@@ -512,7 +512,7 @@ impl Engine {
         let src = src.as_ref().map(|s| s.as_str());
         let context = crate::EvalContext::new(self, global, caches, scope, this_ptr);
 
-        if let Some(ref x) = self.debugger {
+        if let Some(ref x) = self.debugger_interface {
             let (.., ref on_debugger) = **x;
 
             let command = on_debugger(context, event, node, src, node.position())?;

src/eval/global_state.rs

@@ -8,7 +8,7 @@ use std::prelude::v1::*;
 /// Collection of globally-defined constants.
 #[cfg(not(feature = "no_module"))]
 #[cfg(not(feature = "no_function"))]
-pub type GlobalConstants =
+pub type SharedGlobalConstants =
     crate::Shared<crate::Locked<std::collections::BTreeMap<ImmutableString, Dynamic>>>;
 
 /// _(internals)_ Global runtime states.
@@ -67,12 +67,12 @@ pub struct GlobalRuntimeState {
     /// Interior mutability is needed because it is shared in order to aid in cloning.
     #[cfg(not(feature = "no_module"))]
     #[cfg(not(feature = "no_function"))]
-    pub constants: Option<GlobalConstants>,
+    pub constants: Option<SharedGlobalConstants>,
     /// Custom state that can be used by the external host.
     pub tag: Dynamic,
     /// Debugging interface.
     #[cfg(feature = "debugging")]
-    pub(crate) debugger: Option<super::Debugger>,
+    pub(crate) debugger: Option<Box<super::Debugger>>,
 }
 
 impl GlobalRuntimeState {
@@ -103,9 +103,9 @@ impl GlobalRuntimeState {
             tag: engine.default_tag().clone(),
 
             #[cfg(feature = "debugging")]
-            debugger: engine.debugger.as_ref().map(|x| {
+            debugger: engine.debugger_interface.as_ref().map(|x| {
                 let dbg = crate::eval::Debugger::new(crate::eval::DebuggerStatus::Init);
-                (x.0)(engine, dbg)
+                (x.0)(engine, dbg).into()
             }),
         }
     }
@@ -116,7 +116,7 @@ impl GlobalRuntimeState {
     #[inline]
     #[must_use]
     pub fn num_imports(&self) -> usize {
-        self.modules.as_ref().map_or(0, |m| m.len())
+        self.modules.as_deref().map_or(0, crate::StaticVec::len)
     }
     /// Get the globally-imported [module][crate::Module] at a particular index.
     ///
@@ -139,7 +139,7 @@ impl GlobalRuntimeState {
         &mut self,
         index: usize,
     ) -> Option<&mut crate::SharedModule> {
-        self.modules.as_mut().and_then(|m| m.get_mut(index))
+        self.modules.as_deref_mut().and_then(|m| m.get_mut(index))
     }
     /// Get the index of a globally-imported [module][crate::Module] by name.
     ///
@@ -184,8 +184,8 @@ impl GlobalRuntimeState {
             self.imports = None;
             self.modules = None;
         } else if self.imports.is_some() {
-            self.imports.as_mut().unwrap().truncate(size);
+            self.imports.as_deref_mut().unwrap().truncate(size);
-            self.modules.as_mut().unwrap().truncate(size);
+            self.modules.as_deref_mut().unwrap().truncate(size);
         }
     }
     /// Get an iterator to the stack of globally-imported [modules][crate::Module] in reverse order.
@@ -235,7 +235,7 @@ impl GlobalRuntimeState {
     #[cfg(not(feature = "no_module"))]
     #[inline]
     pub(crate) fn may_contain_dynamic_fn(&self, hash_script: u64) -> bool {
-        self.modules.as_ref().map_or(false, |m| {
+        self.modules.as_deref().map_or(false, |m| {
             m.iter().any(|m| m.may_contain_dynamic_fn(hash_script))
         })
     }
@@ -324,7 +324,7 @@ impl GlobalRuntimeState {
     /// Panics if the debugging interface is not set.
     #[cfg(feature = "debugging")]
     pub fn debugger_mut(&mut self) -> &mut super::Debugger {
-        self.debugger.as_mut().unwrap()
+        self.debugger.as_deref_mut().unwrap()
     }
 }
 

src/eval/mod.rs

@@ -17,10 +17,10 @@ pub use debugger::{
     OnDebuggerCallback, OnDebuggingInit,
 };
 pub use eval_context::EvalContext;
+pub use global_state::GlobalRuntimeState;
 #[cfg(not(feature = "no_module"))]
 #[cfg(not(feature = "no_function"))]
-pub use global_state::GlobalConstants;
+pub use global_state::SharedGlobalConstants;
-pub use global_state::GlobalRuntimeState;
 #[cfg(not(feature = "no_index"))]
 pub use target::calc_offset_len;
 pub use target::{calc_index, Target};

src/eval/stmt.rs

@@ -129,7 +129,8 @@ impl Engine {
             let args = &mut [&mut *lock_guard, &mut new_val];
 
             if self.fast_operators() {
-                if let Some(func) = get_builtin_op_assignment_fn(op_assign_token, args[0], args[1])
+                if let Some(func) =
+                    get_builtin_op_assignment_fn(op_assign_token.clone(), args[0], args[1])
                 {
                     // Built-in found
                     let op = op_assign_token.literal_syntax();
@@ -145,7 +146,7 @@ impl Engine {
 
             let op_assign = op_assign_token.literal_syntax();
             let op = op_token.literal_syntax();
-            let token = Some(op_assign_token);
+            let token = op_assign_token.clone();
 
             match self
                 .exec_native_fn_call(global, caches, op_assign, token, hash, args, true, *op_pos)
@@ -154,7 +155,7 @@ impl Engine {
                 Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) =>
                 {
                     // Expand to `var = var op rhs`
-                    let token = Some(op_token);
+                    let token = op_token.clone();
 
                     *args[0] = self
                         .exec_native_fn_call(

src/func/builtin.rs

@@ -84,7 +84,7 @@ fn const_false_fn(_: NativeCallContext, _: &mut [&mut Dynamic]) -> RhaiResult {
 ///
 /// The return function will be registered as a _method_, so the first parameter cannot be consumed.
 #[must_use]
-pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<FnBuiltin> {
+pub fn get_builtin_binary_op_fn(op: Token, x: &Dynamic, y: &Dynamic) -> Option<FnBuiltin> {
     let type1 = x.type_id();
     let type2 = y.type_id();
 
@@ -587,7 +587,7 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
 ///
 /// The return function is registered as a _method_, so the first parameter cannot be consumed.
 #[must_use]
-pub fn get_builtin_op_assignment_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<FnBuiltin> {
+pub fn get_builtin_op_assignment_fn(op: Token, x: &Dynamic, y: &Dynamic) -> Option<FnBuiltin> {
     let type1 = x.type_id();
     let type2 = y.type_id();
 

src/func/call.rs

@@ -165,7 +165,7 @@ impl Engine {
         _global: &GlobalRuntimeState,
         caches: &'s mut Caches,
         local_entry: &'s mut Option<FnResolutionCacheEntry>,
-        op_token: Option<&Token>,
+        op_token: Token,
         hash_base: u64,
         args: Option<&mut FnCallArgs>,
         allow_dynamic: bool,
@@ -174,7 +174,7 @@ impl Engine {
             return None;
         }
 
-        let mut hash = args.as_ref().map_or(hash_base, |args| {
+        let mut hash = args.as_deref().map_or(hash_base, |args| {
             calc_fn_hash_full(hash_base, args.iter().map(|a| a.type_id()))
         });
 
@@ -183,7 +183,7 @@ impl Engine {
         match cache.map.entry(hash) {
             Entry::Occupied(entry) => entry.into_mut().as_ref(),
             Entry::Vacant(entry) => {
-                let num_args = args.as_ref().map_or(0, |a| a.len());
+                let num_args = args.as_deref().map_or(0, |a| a.len());
                 let mut max_bitmask = 0; // One above maximum bitmask based on number of parameters.
                                          // Set later when a specific matching function is not found.
                 let mut bitmask = 1usize; // Bitmask of which parameter to replace with `Dynamic`
@@ -272,7 +272,8 @@ impl Engine {
                         // Try to find a built-in version
                         let builtin =
                             args.and_then(|args| match op_token {
-                                Some(token) if token.is_op_assignment() => {
+                                Token::NonToken => None,
+                                token if token.is_op_assignment() => {
                                     let (first_arg, rest_args) = args.split_first().unwrap();
 
                                     get_builtin_op_assignment_fn(token, first_arg, rest_args[0])
@@ -281,13 +282,11 @@ impl Engine {
                                             source: None,
                                         })
                                 }
-                                Some(token) => get_builtin_binary_op_fn(token, args[0], args[1])
+                                token => get_builtin_binary_op_fn(token.clone(), args[0], args[1])
                                     .map(|f| FnResolutionCacheEntry {
                                         func: CallableFunction::Method(Shared::new(f)),
                                         source: None,
                                     }),
-
-                                None => None,
                             });
 
                         return if cache.filter.is_absent_and_set(hash) {
@@ -340,7 +339,7 @@ impl Engine {
         global: &mut GlobalRuntimeState,
         caches: &mut Caches,
         name: &str,
-        op_token: Option<&Token>,
+        op_token: Token,
         hash: u64,
         args: &mut FnCallArgs,
         is_ref_mut: bool,
@@ -385,7 +384,7 @@ impl Engine {
                 &mut *RestoreOnDrop::lock_if(swap, args, move |a| backup.restore_first_arg(a));
 
             #[cfg(feature = "debugging")]
-            if self.debugger.is_some() {
+            if self.is_debugger_registered() {
                 let source = source.clone().or_else(|| global.source.clone());
 
                 global.debugger_mut().push_call_stack_frame(
@@ -417,7 +416,7 @@ impl Engine {
             };
 
             #[cfg(feature = "debugging")]
-            if self.debugger.is_some() {
+            if self.is_debugger_registered() {
                 use crate::eval::{DebuggerEvent, DebuggerStatus};
 
                 let trigger = match global.debugger().status {
@@ -559,7 +558,7 @@ impl Engine {
         caches: &mut Caches,
         _scope: Option<&mut Scope>,
         fn_name: &str,
-        op_token: Option<&Token>,
+        op_token: Token,
         hashes: FnCallHashes,
         mut _args: &mut FnCallArgs,
         is_ref_mut: bool,
@@ -638,7 +637,15 @@ impl Engine {
             let local_entry = &mut None;
 
             if let Some(FnResolutionCacheEntry { func, ref source }) = self
-                .resolve_fn(global, caches, local_entry, None, hash, None, false)
+                .resolve_fn(
+                    global,
+                    caches,
+                    local_entry,
+                    Token::NonToken,
+                    hash,
+                    None,
+                    false,
+                )
                 .cloned()
             {
                 // Script function call
@@ -721,7 +728,7 @@ impl Engine {
 
         // Do not match function exit for arguments
         #[cfg(feature = "debugging")]
-        let reset = global.debugger.as_mut().and_then(|dbg| {
+        let reset = global.debugger.as_deref_mut().and_then(|dbg| {
             dbg.clear_status_if(|status| {
                 matches!(status, crate::eval::DebuggerStatus::FunctionExit(..))
             })
@@ -782,7 +789,7 @@ impl Engine {
                     caches,
                     None,
                     fn_name,
-                    None,
+                    Token::NonToken,
                     new_hash,
                     &mut args,
                     false,
@@ -836,7 +843,7 @@ impl Engine {
                     caches,
                     None,
                     &fn_name,
-                    None,
+                    Token::NonToken,
                     new_hash,
                     &mut args,
                     is_ref_mut,
@@ -935,7 +942,7 @@ impl Engine {
                     caches,
                     None,
                     fn_name,
-                    None,
+                    Token::NonToken,
                     hash,
                     &mut args,
                     is_ref_mut,
@@ -961,7 +968,7 @@ impl Engine {
         scope: &mut Scope,
         this_ptr: &mut Dynamic,
         fn_name: &str,
-        op_token: Option<&Token>,
+        op_token: Token,
         first_arg: Option<&Expr>,
         args_expr: &[Expr],
         hashes: FnCallHashes,
@@ -977,7 +984,7 @@ impl Engine {
         let redirected; // Handle call() - Redirect function call
 
         match name {
-            _ if op_token.is_some() => (),
+            _ if op_token != Token::NonToken => (),
 
             // Handle call()
             KEYWORD_FN_PTR_CALL if total_args >= 1 => {
@@ -1488,10 +1495,10 @@ impl Engine {
             ..
         } = expr;
 
-        let op_token = op_token.as_ref();
+        let op_token = op_token.clone();
 
         // Short-circuit native binary operator call if under Fast Operators mode
-        if op_token.is_some() && self.fast_operators() && args.len() == 2 {
+        if op_token != Token::NonToken && self.fast_operators() && args.len() == 2 {
             let mut lhs = self
                 .get_arg_value(global, caches, scope, this_ptr, &args[0])?
                 .0
@@ -1504,8 +1511,7 @@ impl Engine {
 
             let operands = &mut [&mut lhs, &mut rhs];
 
-            if let Some(func) =
+            if let Some(func) = get_builtin_binary_op_fn(op_token.clone(), operands[0], operands[1])
-                get_builtin_binary_op_fn(op_token.as_ref().unwrap(), operands[0], operands[1])
             {
                 // Built-in found
                 let orig_level = global.level;

src/func/native.rs

@@ -424,8 +424,7 @@ impl<'a> NativeCallContext<'a> {
         let caches = &mut Caches::new();
 
         let fn_name = fn_name.as_ref();
-        let op_token = Token::lookup_symbol_from_syntax(fn_name);
+        let op_token = Token::lookup_symbol_from_syntax(fn_name).unwrap_or(Token::NonToken);
-        let op_token = op_token.as_ref();
         let args_len = args.len();
 
         global.level += 1;

src/func/script.rs

@@ -69,7 +69,7 @@ impl Engine {
         }
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_none() && fn_def.body.is_empty() {
+        if self.debugger_interface.is_none() && fn_def.body.is_empty() {
             return Ok(Dynamic::UNIT);
         }
         #[cfg(not(feature = "debugging"))]
@@ -96,7 +96,7 @@ impl Engine {
 
         // Push a new call stack frame
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             let source = global.source.clone();
 
             global.debugger_mut().push_call_stack_frame(
@@ -131,7 +131,7 @@ impl Engine {
         };
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             let node = crate::ast::Stmt::Noop(fn_def.body.position());
             self.run_debugger(global, caches, scope, this_ptr, &node)?;
         }
@@ -161,7 +161,7 @@ impl Engine {
             });
 
         #[cfg(feature = "debugging")]
-        if self.debugger.is_some() {
+        if self.is_debugger_registered() {
             let trigger = match global.debugger_mut().status {
                 crate::eval::DebuggerStatus::FunctionExit(n) => n >= global.level,
                 crate::eval::DebuggerStatus::Next(.., true) => true,

src/module/mod.rs

@@ -392,7 +392,7 @@ impl Module {
     #[inline]
     #[must_use]
     pub fn doc(&self) -> &str {
-        self.doc.as_ref().map_or("", |s| s.as_str())
+        self.doc.as_deref().map_or("", SmartString::as_str)
     }
 
     /// Set the documentation of the [`Module`].
@@ -542,16 +542,28 @@ impl Module {
     #[must_use]
     pub fn is_empty(&self) -> bool {
         !self.flags.contains(ModuleFlags::INDEXED_GLOBAL_FUNCTIONS)
-            && self.functions.as_ref().map_or(true, |m| m.is_empty())
+            && self
-            && self.variables.as_ref().map_or(true, |m| m.is_empty())
+                .functions
-            && self.modules.as_ref().map_or(true, |m| m.is_empty())
+                .as_ref()
-            && self.type_iterators.as_ref().map_or(true, |t| t.is_empty())
+                .map_or(true, StraightHashMap::is_empty)
-            && self.all_functions.as_ref().map_or(true, |m| m.is_empty())
+            && self.variables.as_deref().map_or(true, BTreeMap::is_empty)
-            && self.all_variables.as_ref().map_or(true, |m| m.is_empty())
+            && self.modules.as_deref().map_or(true, BTreeMap::is_empty)
+            && self
+                .type_iterators
+                .as_deref()
+                .map_or(true, BTreeMap::is_empty)
+            && self
+                .all_functions
+                .as_deref()
+                .map_or(true, StraightHashMap::is_empty)
+            && self
+                .all_variables
+                .as_deref()
+                .map_or(true, StraightHashMap::is_empty)
             && self
                 .all_type_iterators
-                .as_ref()
+                .as_deref()
-                .map_or(true, |m| m.is_empty())
+                .map_or(true, BTreeMap::is_empty)
     }
 
     /// Is the [`Module`] indexed?
@@ -1633,8 +1645,8 @@ impl Module {
         self.flags &= !ModuleFlags::INDEXED & !ModuleFlags::INDEXED_GLOBAL_FUNCTIONS;
 
         #[cfg(feature = "metadata")]
-        if !other.doc.as_ref().map_or(true, |s| s.is_empty()) {
+        if !other.doc.as_deref().map_or(true, SmartString::is_empty) {
-            if !self.doc.as_ref().map_or(true, |s| s.is_empty()) {
+            if !self.doc.as_deref().map_or(true, SmartString::is_empty) {
                 self.doc.get_or_insert_with(Default::default).push('\n');
             }
             self.doc
@@ -1689,8 +1701,8 @@ impl Module {
         self.flags &= !ModuleFlags::INDEXED & !ModuleFlags::INDEXED_GLOBAL_FUNCTIONS;
 
         #[cfg(feature = "metadata")]
-        if !other.doc.as_ref().map_or(true, |s| s.is_empty()) {
+        if !other.doc.as_deref().map_or(true, SmartString::is_empty) {
-            if !self.doc.as_ref().map_or(true, |s| s.is_empty()) {
+            if !self.doc.as_deref().map_or(true, SmartString::is_empty) {
                 self.doc.get_or_insert_with(Default::default).push('\n');
             }
             self.doc
@@ -1754,8 +1766,8 @@ impl Module {
         self.flags &= !ModuleFlags::INDEXED & !ModuleFlags::INDEXED_GLOBAL_FUNCTIONS;
 
         #[cfg(feature = "metadata")]
-        if !other.doc.as_ref().map_or(true, |s| s.is_empty()) {
+        if !other.doc.as_deref().map_or(true, SmartString::is_empty) {
-            if !self.doc.as_ref().map_or(true, |s| s.is_empty()) {
+            if !self.doc.as_deref().map_or(true, SmartString::is_empty) {
                 self.doc.get_or_insert_with(Default::default).push('\n');
             }
             self.doc
@@ -1837,8 +1849,8 @@ impl Module {
         self.flags &= !ModuleFlags::INDEXED & !ModuleFlags::INDEXED_GLOBAL_FUNCTIONS;
 
         #[cfg(feature = "metadata")]
-        if !other.doc.as_ref().map_or(true, |s| s.is_empty()) {
+        if !other.doc.as_deref().map_or(true, SmartString::is_empty) {
-            if !self.doc.as_ref().map_or(true, |s| s.is_empty()) {
+            if !self.doc.as_deref().map_or(true, SmartString::is_empty) {
                 self.doc.get_or_insert_with(Default::default).push('\n');
             }
             self.doc
@@ -1886,9 +1898,9 @@ impl Module {
     #[must_use]
     pub fn count(&self) -> (usize, usize, usize) {
         (
-            self.variables.as_ref().map_or(0, |m| m.len()),
+            self.variables.as_deref().map_or(0, BTreeMap::len),
-            self.functions.as_ref().map_or(0, |m| m.len()),
+            self.functions.as_ref().map_or(0, StraightHashMap::len),
-            self.type_iterators.as_ref().map_or(0, |t| t.len()),
+            self.type_iterators.as_deref().map_or(0, BTreeMap::len),
         )
     }
 
@@ -2260,11 +2272,11 @@ impl Module {
         if !self.is_indexed() {
             let mut path = Vec::with_capacity(4);
             let mut variables = StraightHashMap::with_capacity_and_hasher(
-                self.variables.as_ref().map_or(0, |m| m.len()),
+                self.variables.as_deref().map_or(0, BTreeMap::len),
                 Default::default(),
             );
             let mut functions = StraightHashMap::with_capacity_and_hasher(
-                self.functions.as_ref().map_or(0, |m| m.len()),
+                self.functions.as_ref().map_or(0, StraightHashMap::len),
                 Default::default(),
             );
             let mut type_iterators = BTreeMap::new();

src/module/resolvers/file.rs

@@ -170,7 +170,7 @@ impl FileModuleResolver {
     #[inline(always)]
     #[must_use]
     pub fn base_path(&self) -> Option<&Path> {
-        self.base_path.as_ref().map(<_>::as_ref)
+        self.base_path.as_deref()
     }
     /// Set the base path for script files.
     #[inline(always)]

src/optimizer.rs

@@ -65,7 +65,7 @@ struct OptimizerState<'a> {
 }
 
 impl<'a> OptimizerState<'a> {
-    /// Create a new State.
+    /// Create a new [`OptimizerState`].
     #[inline(always)]
     pub fn new(
         engine: &'a Engine,
@@ -138,7 +138,7 @@ impl<'a> OptimizerState<'a> {
     pub fn call_fn_with_constant_arguments(
         &mut self,
         fn_name: &str,
-        op_token: Option<&Token>,
+        op_token: Token,
         arg_values: &mut [Dynamic],
     ) -> Dynamic {
         self.engine
@@ -156,39 +156,6 @@ impl<'a> OptimizerState<'a> {
     }
 }
 
-// Has a system function a Rust-native override?
-fn has_native_fn_override(
-    engine: &Engine,
-    hash_script: u64,
-    arg_types: impl AsRef<[TypeId]>,
-) -> bool {
-    let hash = calc_fn_hash_full(hash_script, arg_types.as_ref().iter().copied());
-
-    // First check the global namespace and packages, but skip modules that are standard because
-    // they should never conflict with system functions.
-    if engine
-        .global_modules
-        .iter()
-        .filter(|m| !m.flags.contains(ModuleFlags::STANDARD_LIB))
-        .any(|m| m.contains_fn(hash))
-    {
-        return true;
-    }
-
-    // Then check sub-modules
-    #[cfg(not(feature = "no_module"))]
-    if engine
-        .global_sub_modules
-        .iter()
-        .flat_map(|m| m.values())
-        .any(|m| m.contains_qualified_fn(hash))
-    {
-        return true;
-    }
-
-    false
-}
-
 /// Optimize a block of [statements][Stmt].
 fn optimize_stmt_block(
     mut statements: StmtBlockContainer,
@@ -1150,8 +1117,8 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) {
                     return;
                 }
                 // Overloaded operators can override built-in.
-                _ if x.args.len() == 2 && x.op_token.is_some() && (state.engine.fast_operators() || !has_native_fn_override(state.engine, x.hashes.native(), &arg_types)) => {
+                _ if x.args.len() == 2 && x.op_token != Token::NonToken && (state.engine.fast_operators() || !state.engine.has_native_fn_override(x.hashes.native(), &arg_types)) => {
-                    if let Some(result) = get_builtin_binary_op_fn(x.op_token.as_ref().unwrap(), &arg_values[0], &arg_values[1])
+                    if let Some(result) = get_builtin_binary_op_fn(x.op_token.clone(), &arg_values[0], &arg_values[1])
                         .and_then(|f| {
                             let context = (state.engine, x.name.as_str(),None, &state.global, *pos).into();
                             let (first, second) = arg_values.split_first_mut().unwrap();
@@ -1204,7 +1171,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) {
                     KEYWORD_TYPE_OF if arg_values.len() == 1 => state.engine.map_type_name(arg_values[0].type_name()).into(),
                     #[cfg(not(feature = "no_closure"))]
                     crate::engine::KEYWORD_IS_SHARED if arg_values.len() == 1 => Dynamic::FALSE,
-                    _ => state.call_fn_with_constant_arguments(&x.name, x.op_token.as_ref(), arg_values)
+                    _ => state.call_fn_with_constant_arguments(&x.name, x.op_token.clone(), arg_values)
                 };
 
                 if !result.is_null() {
@@ -1260,124 +1227,149 @@ fn optimize_expr(expr: &mut Expr, state: &mut OptimizerState, _chaining: bool) {
     }
 }
 
-/// Optimize a block of [statements][Stmt] at top level.
+impl Engine {
-///
+    /// Has a system function a Rust-native override?
-/// Constants and variables from the scope are added.
+    fn has_native_fn_override(&self, hash_script: u64, arg_types: impl AsRef<[TypeId]>) -> bool {
-fn optimize_top_level(
+        let hash = calc_fn_hash_full(hash_script, arg_types.as_ref().iter().copied());
-    statements: StmtBlockContainer,
+
-    engine: &Engine,
+        // First check the global namespace and packages, but skip modules that are standard because
-    scope: &Scope,
+        // they should never conflict with system functions.
-    #[cfg(not(feature = "no_function"))] lib: &[crate::SharedModule],
+        if self
-    optimization_level: OptimizationLevel,
+            .global_modules
-) -> StmtBlockContainer {
+            .iter()
-    let mut statements = statements;
+            .filter(|m| !m.flags.contains(ModuleFlags::STANDARD_LIB))
+            .any(|m| m.contains_fn(hash))
+        {
+            return true;
+        }
+
+        // Then check sub-modules
+        #[cfg(not(feature = "no_module"))]
+        if self
+            .global_sub_modules
+            .iter()
+            .flat_map(|m| m.values())
+            .any(|m| m.contains_qualified_fn(hash))
+        {
+            return true;
+        }
+
+        false
+    }
+
+    /// Optimize a block of [statements][Stmt] at top level.
+    ///
+    /// Constants and variables from the scope are added.
+    fn optimize_top_level(
+        &self,
+        statements: StmtBlockContainer,
+        scope: &Scope,
+        #[cfg(not(feature = "no_function"))] lib: &[crate::SharedModule],
+        optimization_level: OptimizationLevel,
+    ) -> StmtBlockContainer {
+        let mut statements = statements;
+
+        // If optimization level is None then skip optimizing
+        if optimization_level == OptimizationLevel::None {
+            statements.shrink_to_fit();
+            return statements;
+        }
+
+        // Set up the state
+        let mut state = OptimizerState::new(
+            self,
+            #[cfg(not(feature = "no_function"))]
+            lib,
+            optimization_level,
+        );
+
+        // Add constants from global modules
+        for (name, value) in self.global_modules.iter().rev().flat_map(|m| m.iter_var()) {
+            state.push_var(name, AccessMode::ReadOnly, value.clone());
+        }
+
+        // Add constants and variables from the scope
+        for (name, constant, value) in scope.iter() {
+            if constant {
+                state.push_var(name, AccessMode::ReadOnly, value);
+            } else {
+                state.push_var(name, AccessMode::ReadWrite, Dynamic::NULL);
+            }
+        }
+
+        optimize_stmt_block(statements, &mut state, true, false, true)
+    }
+
+    /// Optimize a collection of statements and functions into an [`AST`].
+    pub(crate) fn optimize_into_ast(
+        &self,
+        scope: &Scope,
+        statements: StmtBlockContainer,
+        #[cfg(not(feature = "no_function"))] functions: StaticVec<
+            crate::Shared<crate::ast::ScriptFnDef>,
+        >,
+        optimization_level: OptimizationLevel,
+    ) -> AST {
+        let mut statements = statements;
+
+        #[cfg(not(feature = "no_function"))]
+        let lib: crate::Shared<_> = {
+            let mut module = crate::Module::new();
+
+            if optimization_level != OptimizationLevel::None {
+                // We only need the script library's signatures for optimization purposes
+                let mut lib2 = crate::Module::new();
+
+                for fn_def in &functions {
+                    lib2.set_script_fn(crate::ast::ScriptFnDef {
+                        name: fn_def.name.clone(),
+                        access: fn_def.access,
+                        body: crate::ast::StmtBlock::NONE,
+                        params: fn_def.params.clone(),
+                        #[cfg(not(feature = "no_module"))]
+                        environ: None,
+                        #[cfg(not(feature = "no_function"))]
+                        #[cfg(feature = "metadata")]
+                        comments: Box::default(),
+                    });
+                }
+
+                let lib2 = &[lib2.into()];
+
+                for fn_def in functions {
+                    let mut fn_def = crate::func::shared_take_or_clone(fn_def);
+
+                    // Optimize the function body
+                    let body = mem::take(&mut *fn_def.body);
+
+                    *fn_def.body = self.optimize_top_level(body, scope, lib2, optimization_level);
+
+                    module.set_script_fn(fn_def);
+                }
+            } else {
+                for fn_def in functions {
+                    module.set_script_fn(fn_def);
+                }
+            }
+
+            module.into()
+        };
 
-    // If optimization level is None then skip optimizing
-    if optimization_level == OptimizationLevel::None {
         statements.shrink_to_fit();
-        return statements;
-    }
 
-    // Set up the state
+        AST::new(
-    let mut state = OptimizerState::new(
+            match optimization_level {
-        engine,
+                OptimizationLevel::None => statements,
-        #[cfg(not(feature = "no_function"))]
+                OptimizationLevel::Simple | OptimizationLevel::Full => self.optimize_top_level(
-        lib,
+                    statements,
-        optimization_level,
+                    scope,
-    );
-
-    // Add constants from global modules
-    for (name, value) in engine
-        .global_modules
-        .iter()
-        .rev()
-        .flat_map(|m| m.iter_var())
-    {
-        state.push_var(name, AccessMode::ReadOnly, value.clone());
-    }
-
-    // Add constants and variables from the scope
-    for (name, constant, value) in scope.iter() {
-        if constant {
-            state.push_var(name, AccessMode::ReadOnly, value);
-        } else {
-            state.push_var(name, AccessMode::ReadWrite, Dynamic::NULL);
-        }
-    }
-
-    optimize_stmt_block(statements, &mut state, true, false, true)
-}
-
-/// Optimize an [`AST`].
-pub fn optimize_into_ast(
-    engine: &Engine,
-    scope: &Scope,
-    statements: StmtBlockContainer,
-    #[cfg(not(feature = "no_function"))] functions: StaticVec<
-        crate::Shared<crate::ast::ScriptFnDef>,
-    >,
-    optimization_level: OptimizationLevel,
-) -> AST {
-    let mut statements = statements;
-
-    #[cfg(not(feature = "no_function"))]
-    let lib: crate::Shared<_> = {
-        let mut module = crate::Module::new();
-
-        if optimization_level != OptimizationLevel::None {
-            // We only need the script library's signatures for optimization purposes
-            let mut lib2 = crate::Module::new();
-
-            for fn_def in &functions {
-                lib2.set_script_fn(crate::ast::ScriptFnDef {
-                    name: fn_def.name.clone(),
-                    access: fn_def.access,
-                    body: crate::ast::StmtBlock::NONE,
-                    params: fn_def.params.clone(),
-                    #[cfg(not(feature = "no_module"))]
-                    environ: None,
                     #[cfg(not(feature = "no_function"))]
-                    #[cfg(feature = "metadata")]
+                    &[lib.clone()],
-                    comments: Box::default(),
+                    optimization_level,
-                });
+                ),
-            }
+            },
-
+            #[cfg(not(feature = "no_function"))]
-            let lib2 = &[lib2.into()];
+            lib,
-
+        )
-            for fn_def in functions {
+    }
-                let mut fn_def = crate::func::shared_take_or_clone(fn_def);
-
-                // Optimize the function body
-                let body = mem::take(&mut *fn_def.body);
-
-                *fn_def.body = optimize_top_level(body, engine, scope, lib2, optimization_level);
-
-                module.set_script_fn(fn_def);
-            }
-        } else {
-            for fn_def in functions {
-                module.set_script_fn(fn_def);
-            }
-        }
-
-        module.into()
-    };
-
-    statements.shrink_to_fit();
-
-    AST::new(
-        match optimization_level {
-            OptimizationLevel::None => statements,
-            OptimizationLevel::Simple | OptimizationLevel::Full => optimize_top_level(
-                statements,
-                engine,
-                scope,
-                #[cfg(not(feature = "no_function"))]
-                &[lib.clone()],
-                optimization_level,
-            ),
-        },
-        #[cfg(not(feature = "no_function"))]
-        lib,
-    )
 }

src/parser.rs

@@ -607,7 +607,7 @@ impl Engine {
                             .any(|m| m.as_str() == root)
                         && !self
                             .global_sub_modules
-                            .as_ref()
+                            .as_deref()
                             .map_or(false, |m| m.contains_key(root))
                     {
                         return Err(
@@ -633,7 +633,7 @@ impl Engine {
                 return Ok(FnCallExpr {
                     name: state.get_interned_string(id),
                     capture_parent_scope,
-                    op_token: None,
+                    op_token: Token::NonToken,
                     #[cfg(not(feature = "no_module"))]
                     namespace,
                     hashes,
@@ -682,7 +682,7 @@ impl Engine {
                                 .any(|m| m.as_str() == root)
                             && !self
                                 .global_sub_modules
-                                .as_ref()
+                                .as_deref()
                                 .map_or(false, |m| m.contains_key(root))
                         {
                             return Err(
@@ -708,7 +708,7 @@ impl Engine {
                     return Ok(FnCallExpr {
                         name: state.get_interned_string(id),
                         capture_parent_scope,
-                        op_token: None,
+                        op_token: Token::NonToken,
                         #[cfg(not(feature = "no_module"))]
                         namespace,
                         hashes,
@@ -1413,8 +1413,11 @@ impl Engine {
             Token::Pipe | Token::Or if settings.has_option(LangOptions::ANON_FN) => {
                 // Build new parse state
                 let new_interner = &mut StringsInterner::new();
-                let mut new_state =
+                let new_state = &mut ParseState::new(
-                    ParseState::new(state.scope, new_interner, state.tokenizer_control.clone());
+                    state.scope,
+                    new_interner,
+                    state.tokenizer_control.clone(),
+                );
 
                 // We move the strings interner to the new parse state object by swapping it...
                 std::mem::swap(state.interned_strings, new_state.interned_strings);
@@ -1455,7 +1458,7 @@ impl Engine {
                     ..settings
                 };
 
-                let result = self.parse_anon_fn(input, &mut new_state, state, lib, new_settings);
+                let result = self.parse_anon_fn(input, new_state, state, lib, new_settings);
 
                 // Restore the strings interner by swapping it back
                 std::mem::swap(state.interned_strings, new_state.interned_strings);
@@ -1568,12 +1571,12 @@ impl Engine {
             Token::Custom(key) | Token::Reserved(key) | Token::Identifier(key)
                 if self
                     .custom_syntax
-                    .as_ref()
+                    .as_deref()
                     .map_or(false, |m| m.contains_key(&**key)) =>
             {
                 let (key, syntax) = self
                     .custom_syntax
-                    .as_ref()
+                    .as_deref()
                     .and_then(|m| m.get_key_value(&**key))
                     .unwrap();
                 let (.., pos) = input.next().expect(NEVER_ENDS);
@@ -1885,7 +1888,7 @@ impl Engine {
                             .any(|m| m.as_str() == root)
                         && !self
                             .global_sub_modules
-                            .as_ref()
+                            .as_deref()
                             .map_or(false, |m| m.contains_key(root))
                     {
                         return Err(
@@ -1954,7 +1957,7 @@ impl Engine {
                             name: state.get_interned_string("-"),
                             hashes: FnCallHashes::from_native(calc_fn_hash(None, "-", 1)),
                             args,
-                            op_token: Some(token),
+                            op_token: token,
                             capture_parent_scope: false,
                         }
                         .into_fn_call_expr(pos))
@@ -1983,7 +1986,7 @@ impl Engine {
                             name: state.get_interned_string("+"),
                             hashes: FnCallHashes::from_native(calc_fn_hash(None, "+", 1)),
                             args,
-                            op_token: Some(token),
+                            op_token: token,
                             capture_parent_scope: false,
                         }
                         .into_fn_call_expr(pos))
@@ -2005,7 +2008,7 @@ impl Engine {
                     name: state.get_interned_string("!"),
                     hashes: FnCallHashes::from_native(calc_fn_hash(None, "!", 1)),
                     args,
-                    op_token: Some(token),
+                    op_token: token,
                     capture_parent_scope: false,
                 }
                 .into_fn_call_expr(pos))
@@ -2019,7 +2022,7 @@ impl Engine {
 
     /// Make an assignment statement.
     fn make_assignment_stmt(
-        op: Option<Token>,
+        op: Token,
         state: &mut ParseState,
         lhs: Expr,
         rhs: Expr,
@@ -2052,7 +2055,7 @@ impl Engine {
             }
         }
 
-        let op_info = if let Some(ref op) = op {
+        let op_info = if op != Token::NonToken {
             OpAssignment::new_op_assignment_from_token(op, op_pos)
         } else {
             OpAssignment::new_assignment(op_pos)
@@ -2133,12 +2136,11 @@ impl Engine {
     ) -> ParseResult<Stmt> {
         let (op, pos) = match input.peek().expect(NEVER_ENDS) {
             // var = ...
-            (Token::Equals, ..) => (None, eat_token(input, Token::Equals)),
+            (Token::Equals, ..) => (Token::NonToken, eat_token(input, Token::Equals)),
             // var op= ...
-            (token, ..) if token.is_op_assignment() => input
+            (token, ..) if token.is_op_assignment() => {
-                .next()
+                input.next().map(|(op, pos)| (op, pos)).expect(NEVER_ENDS)
-                .map(|(op, pos)| (Some(op), pos))
+            }
-                .expect(NEVER_ENDS),
             // Not op-assignment
             _ => return Ok(Stmt::Expr(lhs.into())),
         };
@@ -2327,7 +2329,7 @@ impl Engine {
                 #[cfg(not(feature = "no_custom_syntax"))]
                 Token::Custom(c) => self
                     .custom_keywords
-                    .as_ref()
+                    .as_deref()
                     .and_then(|m| m.get(&**c))
                     .copied()
                     .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*current_pos))?,
@@ -2353,7 +2355,7 @@ impl Engine {
                 #[cfg(not(feature = "no_custom_syntax"))]
                 Token::Custom(c) => self
                     .custom_keywords
-                    .as_ref()
+                    .as_deref()
                     .and_then(|m| m.get(&**c))
                     .copied()
                     .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*next_pos))?,
@@ -2380,9 +2382,9 @@ impl Engine {
             let hash = calc_fn_hash(None, &op, 2);
             let is_valid_script_function = is_valid_function_name(&op);
             let operator_token = if is_valid_script_function {
-                None
+                Token::NonToken
             } else {
-                Some(op_token.clone())
+                op_token.clone()
             };
 
             let mut args = StaticVec::new_const();
@@ -2446,7 +2448,7 @@ impl Engine {
                 Token::Custom(s)
                     if self
                         .custom_keywords
-                        .as_ref()
+                        .as_deref()
                         .and_then(|m| m.get(s.as_str()))
                         .map_or(false, Option::is_some) =>
                 {
@@ -2875,7 +2877,7 @@ impl Engine {
         settings.flags |= ParseSettingFlags::BREAKABLE;
         let body = self.parse_block(input, state, lib, settings.level_up()?)?;
 
-        state.stack.as_mut().unwrap().rewind(prev_stack_len);
+        state.stack.as_deref_mut().unwrap().rewind(prev_stack_len);
 
         Ok(Stmt::For(
             Box::new((loop_var, counter_var, expr, body.into())),
@@ -2956,7 +2958,7 @@ impl Engine {
 
         let (existing, hit_barrier) = state.find_var(&name);
 
-        let stack = state.stack.as_mut().unwrap();
+        let stack = state.stack.as_deref_mut().unwrap();
 
         let existing = if !hit_barrier && existing > 0 {
             let offset = stack.len() - existing;
@@ -3072,7 +3074,7 @@ impl Engine {
                 pos: id_pos,
             },
             Ident {
-                name: state.get_interned_string(alias.as_ref().map_or("", <_>::as_ref)),
+                name: state.get_interned_string(alias.as_deref().unwrap_or("")),
                 pos: alias_pos,
             },
         );
@@ -3121,10 +3123,10 @@ impl Engine {
         }
 
         let prev_entry_stack_len = state.block_stack_len;
-        state.block_stack_len = state.stack.as_ref().map_or(0, |s| s.len());
+        state.block_stack_len = state.stack.as_deref().map_or(0, Scope::len);
 
         #[cfg(not(feature = "no_module"))]
-        let orig_imports_len = state.imports.as_ref().map_or(0, |m| m.len());
+        let orig_imports_len = state.imports.as_deref().map_or(0, StaticVec::len);
 
         let end_pos = loop {
             // Terminated?
@@ -3293,7 +3295,7 @@ impl Engine {
                 match input.next().expect(NEVER_ENDS) {
                     (Token::Fn, pos) => {
                         // Build new parse state
-                        let mut new_state = ParseState::new(
+                        let new_state = &mut ParseState::new(
                             state.scope,
                             state.interned_strings,
                             state.tokenizer_control.clone(),
@@ -3330,7 +3332,7 @@ impl Engine {
 
                         let func = self.parse_fn(
                             input,
-                            &mut new_state,
+                            new_state,
                             lib,
                             access,
                             new_settings,
@@ -3521,7 +3523,7 @@ impl Engine {
 
         if !catch_var.is_empty() {
             // Remove the error variable from the stack
-            state.stack.as_mut().unwrap().pop();
+            state.stack.as_deref_mut().unwrap().pop();
         }
 
         Ok(Stmt::TryCatch(
@@ -3684,7 +3686,7 @@ impl Engine {
                 num_externals + 1,
             )),
             args,
-            op_token: None,
+            op_token: Token::NonToken,
             capture_parent_scope: false,
         }
         .into_fn_call_expr(pos);
@@ -3820,7 +3822,7 @@ impl Engine {
     /// Parse a global level expression.
     pub(crate) fn parse_global_expr(
         &self,
-        input: &mut TokenStream,
+        mut input: TokenStream,
         state: &mut ParseState,
         process_settings: impl FnOnce(&mut ParseSettings),
         _optimization_level: OptimizationLevel,
@@ -3842,7 +3844,7 @@ impl Engine {
         };
         process_settings(&mut settings);
 
-        let expr = self.parse_expr(input, state, &mut functions, settings)?;
+        let expr = self.parse_expr(&mut input, state, &mut functions, settings)?;
 
         assert!(functions.is_empty());
 
@@ -3856,8 +3858,7 @@ impl Engine {
         statements.push(Stmt::Expr(expr.into()));
 
         #[cfg(not(feature = "no_optimize"))]
-        return Ok(crate::optimizer::optimize_into_ast(
+        return Ok(self.optimize_into_ast(
-            self,
             state.scope,
             statements,
             #[cfg(not(feature = "no_function"))]
@@ -3876,7 +3877,7 @@ impl Engine {
     /// Parse the global level statements.
     fn parse_global_level(
         &self,
-        input: &mut TokenStream,
+        mut input: TokenStream,
         state: &mut ParseState,
         process_settings: impl FnOnce(&mut ParseSettings),
     ) -> ParseResult<(StmtBlockContainer, StaticVec<Shared<ScriptFnDef>>)> {
@@ -3893,8 +3894,8 @@ impl Engine {
         };
         process_settings(&mut settings);
 
-        while !input.peek().expect(NEVER_ENDS).0.is_eof() {
+        while input.peek().expect(NEVER_ENDS).0 != Token::EOF {
-            let stmt = self.parse_stmt(input, state, &mut functions, settings)?;
+            let stmt = self.parse_stmt(&mut input, state, &mut functions, settings)?;
 
             if stmt.is_noop() {
                 continue;
@@ -3909,7 +3910,7 @@ impl Engine {
                 (Token::EOF, ..) => break,
                 // stmt ;
                 (Token::SemiColon, ..) if need_semicolon => {
-                    eat_token(input, Token::SemiColon);
+                    eat_token(&mut input, Token::SemiColon);
                 }
                 // stmt ;
                 (Token::SemiColon, ..) if !need_semicolon => (),
@@ -3936,15 +3937,14 @@ impl Engine {
     #[inline]
     pub(crate) fn parse(
         &self,
-        input: &mut TokenStream,
+        input: TokenStream,
         state: &mut ParseState,
         _optimization_level: OptimizationLevel,
     ) -> ParseResult<AST> {
         let (statements, _lib) = self.parse_global_level(input, state, |_| {})?;
 
         #[cfg(not(feature = "no_optimize"))]
-        return Ok(crate::optimizer::optimize_into_ast(
+        return Ok(self.optimize_into_ast(
-            self,
             state.scope,
             statements,
             #[cfg(not(feature = "no_function"))]

src/tokenizer.rs

@@ -584,7 +584,10 @@ pub enum Token {
     #[cfg(not(feature = "no_custom_syntax"))]
     Custom(Box<SmartString>),
     /// End of the input stream.
+    /// Used as a placeholder for the end of input.
     EOF,
+    /// Placeholder to indicate the lack of a token.
+    NonToken,
 }
 
 impl fmt::Display for Token {
@@ -610,6 +613,7 @@ impl fmt::Display for Token {
             Comment(s) => f.write_str(s),
 
             EOF => f.write_str("{EOF}"),
+            NonToken => f.write_str("{NONE}"),
 
             token => f.write_str(token.literal_syntax()),
         }
@@ -638,7 +642,7 @@ impl Token {
             Custom(..) => false,
             LexError(..) | Comment(..) => false,
 
-            EOF => false,
+            EOF | NonToken => false,
 
             _ => true,
         }
@@ -956,13 +960,6 @@ impl Token {
         }
     }
 
-    /// Is this token [`EOF`][Token::EOF]?
-    #[inline(always)]
-    #[must_use]
-    pub const fn is_eof(&self) -> bool {
-        matches!(self, Self::EOF)
-    }
-
     /// If another operator is after these, it's probably a unary operator
     /// (not sure about `fn` name).
     #[must_use]
@@ -2435,7 +2432,7 @@ impl<'a> Iterator for TokenIterator<'a> {
             Some((Token::Reserved(s), pos)) => (match
                 (s.as_str(),
                     #[cfg(not(feature = "no_custom_syntax"))]
-                    self.engine.custom_keywords.as_ref().map_or(false, |m| m.contains_key(&*s)),
+                    self.engine.custom_keywords.as_deref().map_or(false, |m| m.contains_key(&*s)),
                     #[cfg(feature = "no_custom_syntax")]
                     false
                 )
@@ -2472,7 +2469,7 @@ impl<'a> Iterator for TokenIterator<'a> {
                 #[cfg(feature = "no_custom_syntax")]
                 (.., true) => unreachable!("no custom operators"),
                 // Reserved keyword that is not custom and disabled.
-                (token, false) if self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token)) => {
+                (token, false) if self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token)) => {
                     let msg = format!("reserved {} '{token}' is disabled", if is_valid_identifier(token) { "keyword"} else {"symbol"});
                     Token::LexError(LERR::ImproperSymbol(s.to_string(), msg).into())
                 },
@@ -2481,13 +2478,13 @@ impl<'a> Iterator for TokenIterator<'a> {
             }, pos),
             // Custom keyword
             #[cfg(not(feature = "no_custom_syntax"))]
-            Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.as_ref().map_or(false,|m| m.contains_key(&*s)) => {
+            Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.as_deref().map_or(false,|m| m.contains_key(&*s)) => {
                 (Token::Custom(s), pos)
             }
             // Custom keyword/symbol - must be disabled
             #[cfg(not(feature = "no_custom_syntax"))]
-            Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.as_ref().map_or(false,|m| m.contains_key(token.literal_syntax())) => {
+            Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.as_deref().map_or(false,|m| m.contains_key(token.literal_syntax())) => {
-                if self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token.literal_syntax())) {
+                if self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token.literal_syntax())) {
                     // Disabled standard keyword/symbol
                     (Token::Custom(Box::new(token.literal_syntax().into())), pos)
                 } else {
@@ -2496,7 +2493,7 @@ impl<'a> Iterator for TokenIterator<'a> {
                 }
             }
             // Disabled symbol
-            Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.as_ref().map_or(false,|m| m.contains(token.literal_syntax())) => {
+            Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.as_deref().map_or(false,|m| m.contains(token.literal_syntax())) => {
                 (Token::Reserved(Box::new(token.literal_syntax().into())), pos)
             }
             // Normal symbol