Use SmartString for buffers and literal_syntax can panic.

src/api/mod.rs

@@ -163,25 +163,18 @@ impl Engine {
             // Active standard keywords cannot be made custom
             // Disabled keywords are OK
             Some(token) if token.is_standard_keyword() => {
-                if self.disabled_symbols.is_empty()
+                if !self.disabled_symbols.contains(token.literal_syntax()) {
-                    || !self.disabled_symbols.contains(&*token.syntax())
-                {
                     return Err(format!("'{keyword}' is a reserved keyword"));
                 }
             }
             // Active standard symbols cannot be made custom
             Some(token) if token.is_standard_symbol() => {
-                if self.disabled_symbols.is_empty()
+                if !self.disabled_symbols.contains(token.literal_syntax()) {
-                    || !self.disabled_symbols.contains(&*token.syntax())
-                {
                     return Err(format!("'{keyword}' is a reserved operator"));
                 }
             }
             // Active standard symbols cannot be made custom
-            Some(token)
+            Some(token) if !self.disabled_symbols.contains(token.literal_syntax()) => {
-                if self.disabled_symbols.is_empty()
-                    || !self.disabled_symbols.contains(&*token.syntax()) =>
-            {
                 return Err(format!("'{keyword}' is a reserved symbol"))
             }
             // Disabled symbols are OK

src/api/register.rs

@@ -688,8 +688,7 @@ impl Engine {
             name: &str,
             module: SharedModule,
         ) {
-            let separator = crate::tokenizer::Token::DoubleColon.syntax();
+            let separator = crate::tokenizer::Token::DoubleColon.literal_syntax();
-            let separator = separator.as_ref();
 
             if name.contains(separator) {
                 let mut iter = name.splitn(2, separator);

src/ast/expr.rs

@@ -361,7 +361,7 @@ impl fmt::Debug for Expr {
     #[cold]
     #[inline(never)]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut display_pos = format!(" @ {:?}", self.start_position());
+        let mut display_pos = self.start_position();
 
         match self {
             Self::DynamicConstant(value, ..) => write!(f, "{value:?}"),
@@ -395,7 +395,7 @@ impl fmt::Debug for Expr {
                     write!(f, "{}{}", x.1, Token::DoubleColon.literal_syntax())?;
                     let pos = x.1.position();
                     if !pos.is_none() {
-                        display_pos = format!(" @ {pos:?}");
+                        display_pos = pos;
                     }
                 }
                 f.write_str(&x.3)?;
@@ -413,7 +413,7 @@ impl fmt::Debug for Expr {
             Self::Stmt(x) => {
                 let pos = x.span();
                 if !pos.is_none() {
-                    display_pos = format!(" @ {pos:?}");
+                    display_pos = pos.start();
                 }
                 f.write_str("ExprStmtBlock")?;
                 f.debug_list().entries(x.iter()).finish()
@@ -421,7 +421,7 @@ impl fmt::Debug for Expr {
             Self::FnCall(x, ..) => fmt::Debug::fmt(x, f),
             Self::Index(x, options, pos) => {
                 if !pos.is_none() {
-                    display_pos = format!(" @ {pos:?}");
+                    display_pos = *pos;
                 }
 
                 let mut f = f.debug_struct("Index");
@@ -434,7 +434,7 @@ impl fmt::Debug for Expr {
             }
             Self::Dot(x, options, pos) => {
                 if !pos.is_none() {
-                    display_pos = format!(" @ {pos:?}");
+                    display_pos = *pos;
                 }
 
                 let mut f = f.debug_struct("Dot");
@@ -454,7 +454,7 @@ impl fmt::Debug for Expr {
                 };
 
                 if !pos.is_none() {
-                    display_pos = format!(" @ {pos:?}");
+                    display_pos = *pos;
                 }
 
                 f.debug_struct(op_name)
@@ -466,7 +466,7 @@ impl fmt::Debug for Expr {
             Self::Custom(x, ..) => f.debug_tuple("Custom").field(x).finish(),
         }?;
 
-        f.write_str(&display_pos)
+        write!(f, " @ {display_pos:?}")
     }
 }
 

src/eval/chaining.rs

@@ -317,7 +317,7 @@ impl Engine {
         scope: &mut Scope,
         this_ptr: &mut Dynamic,
         expr: &Expr,
-        new_val: &mut Option<(Dynamic, &OpAssignment)>,
+        new_val: Option<(Dynamic, &OpAssignment)>,
     ) -> RhaiResult {
         let chain_type = ChainType::from(expr);
 
@@ -507,7 +507,7 @@ impl Engine {
         target: &mut Target,
         rhs: &Expr,
         idx_values: &mut FnArgsVec<Dynamic>,
-        new_val: &mut Option<(Dynamic, &OpAssignment)>,
+        new_val: Option<(Dynamic, &OpAssignment)>,
     ) -> RhaiResultOf<(Dynamic, bool)> {
         let is_ref_mut = target.is_ref();
         let op_pos = parent.position();
@@ -576,7 +576,7 @@ impl Engine {
                         #[cfg(feature = "debugging")]
                         self.run_debugger(global, caches, scope, this_ptr, parent)?;
 
-                        let (new_val, op_info) = new_val.take().expect("`Some`");
+                        let (new_val, op_info) = new_val.expect("`Some`");
                         let idx_val = &mut idx_values.pop().unwrap();
                         let idx = &mut idx_val.clone();
 
@@ -686,12 +686,12 @@ impl Engine {
                         unreachable!("function call in dot chain should not be namespace-qualified")
                     }
                     // {xxx:map}.id op= ???
-                    Expr::Property(x, pos) if target.is_map() && new_val.is_some() => {
+                    Expr::Property(x, pos) if new_val.is_some() && target.is_map() => {
                         #[cfg(feature = "debugging")]
                         self.run_debugger(global, caches, scope, this_ptr, rhs)?;
 
                         let index = &mut x.2.clone().into();
-                        let (new_val, op_info) = new_val.take().expect("`Some`");
+                        let (new_val, op_info) = new_val.expect("`Some`");
                         {
                             let val_target = &mut self.get_indexed_mut(
                                 global, caches, target, index, *pos, op_pos, true, false,
@@ -720,7 +720,7 @@ impl Engine {
                         self.run_debugger(global, caches, scope, this_ptr, rhs)?;
 
                         let ((getter, hash_get), (setter, hash_set), name) = &**x;
-                        let (mut new_val, op_info) = new_val.take().expect("`Some`");
+                        let (mut new_val, op_info) = new_val.expect("`Some`");
 
                         if op_info.is_op_assignment() {
                             let args = &mut [target.as_mut()];

src/eval/data_check.rs

@@ -8,7 +8,7 @@ use std::borrow::Borrow;
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 
-impl Engine {
+impl Dynamic {
     /// Recursively calculate the sizes of a value.
     ///
     /// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`.
@@ -16,20 +16,20 @@ impl Engine {
     /// # Panics
     ///
     /// Panics if any interior data is shared (should never happen).
-    pub(crate) fn calc_data_sizes(value: &Dynamic, _top: bool) -> (usize, usize, usize) {
+    pub(crate) fn calc_data_sizes(&self, _top: bool) -> (usize, usize, usize) {
-        match value.0 {
+        match self.0 {
             #[cfg(not(feature = "no_index"))]
             Union::Array(ref arr, ..) => {
                 arr.iter()
                     .fold((0, 0, 0), |(ax, mx, sx), value| match value.0 {
                         Union::Array(..) => {
-                            let (a, m, s) = Self::calc_data_sizes(value, false);
+                            let (a, m, s) = value.calc_data_sizes(false);
                             (ax + a + 1, mx + m, sx + s)
                         }
                         Union::Blob(ref a, ..) => (ax + 1 + a.len(), mx, sx),
                         #[cfg(not(feature = "no_object"))]
                         Union::Map(..) => {
-                            let (a, m, s) = Self::calc_data_sizes(value, false);
+                            let (a, m, s) = value.calc_data_sizes(false);
                             (ax + a + 1, mx + m, sx + s)
                         }
                         Union::Str(ref s, ..) => (ax + 1, mx, sx + s.len()),
@@ -44,13 +44,13 @@ impl Engine {
                     .fold((0, 0, 0), |(ax, mx, sx), value| match value.0 {
                         #[cfg(not(feature = "no_index"))]
                         Union::Array(..) => {
-                            let (a, m, s) = Self::calc_data_sizes(value, false);
+                            let (a, m, s) = value.calc_data_sizes(false);
                             (ax + a, mx + m + 1, sx + s)
                         }
                         #[cfg(not(feature = "no_index"))]
                         Union::Blob(ref a, ..) => (ax + a.len(), mx, sx),
                         Union::Map(..) => {
-                            let (a, m, s) = Self::calc_data_sizes(value, false);
+                            let (a, m, s) = value.calc_data_sizes(false);
                             (ax + a, mx + m + 1, sx + s)
                         }
                         Union::Str(ref s, ..) => (ax, mx + 1, sx + s.len()),
@@ -59,17 +59,17 @@ impl Engine {
             }
             Union::Str(ref s, ..) => (0, 0, s.len()),
             #[cfg(not(feature = "no_closure"))]
-            Union::Shared(..) if _top => {
+            Union::Shared(..) if _top => self.read_lock::<Dynamic>().unwrap().calc_data_sizes(true),
-                Self::calc_data_sizes(&*value.read_lock::<Dynamic>().unwrap(), true)
-            }
             #[cfg(not(feature = "no_closure"))]
             Union::Shared(..) => {
-                unreachable!("shared values discovered within data: {}", value)
+                unreachable!("shared values discovered within data: {}", self)
             }
             _ => (0, 0, 0),
         }
     }
+}
 
+impl Engine {
     /// Raise an error if any data size exceeds limit.
     ///
     /// [`Position`] in [`EvalAltResult`][crate::EvalAltResult] is always [`NONE`][Position::NONE]
@@ -125,7 +125,7 @@ impl Engine {
             return Ok(value);
         }
 
-        let sizes = Self::calc_data_sizes(value.borrow(), true);
+        let sizes = value.borrow().calc_data_sizes(true);
 
         self.raise_err_if_over_data_size_limit(sizes)
             .map(|_| value)

src/eval/expr.rs

@@ -307,7 +307,7 @@ impl Engine {
 
                             #[cfg(not(feature = "unchecked"))]
                             if self.has_data_size_limit() {
-                                let val_sizes = Self::calc_data_sizes(&value, true);
+                                let val_sizes = value.calc_data_sizes(true);
 
                                 total_data_sizes = (
                                     total_data_sizes.0 + val_sizes.0,
@@ -339,7 +339,7 @@ impl Engine {
 
                         #[cfg(not(feature = "unchecked"))]
                         if self.has_data_size_limit() {
-                            let delta = Self::calc_data_sizes(&value, true);
+                            let delta = value.calc_data_sizes(true);
                             total_data_sizes = (
                                 total_data_sizes.0 + delta.0,
                                 total_data_sizes.1 + delta.1,
@@ -411,13 +411,11 @@ impl Engine {
 
             #[cfg(not(feature = "no_index"))]
             Expr::Index(..) => {
-                self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, &mut None)
+                self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, None)
             }
 
             #[cfg(not(feature = "no_object"))]
-            Expr::Dot(..) => {
+            Expr::Dot(..) => self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, None),
-                self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, &mut None)
-            }
 
             _ => unreachable!("expression cannot be evaluated: {:?}", expr),
         }

src/eval/mod.rs

@@ -21,7 +21,9 @@ pub use eval_context::EvalContext;
 #[cfg(not(feature = "no_function"))]
 pub use global_state::GlobalConstants;
 pub use global_state::GlobalRuntimeState;
-pub use target::{calc_index, calc_offset_len, Target};
+#[cfg(not(feature = "no_index"))]
+pub use target::calc_offset_len;
+pub use target::{calc_index, Target};
 
 #[cfg(feature = "unchecked")]
 mod unchecked {

src/eval/stmt.rs

@@ -258,7 +258,7 @@ impl Engine {
                     rhs_val = self.get_interned_string(value).into();
                 }
 
-                let _new_val = &mut Some((rhs_val, op_info));
+                let _new_val = Some((rhs_val, op_info));
 
                 // Must be either `var[index] op= val` or `var.prop op= val`
                 match lhs {

src/eval/target.rs

@@ -13,8 +13,8 @@ use std::{
 /// Negative starting positions count from the end.
 ///
 /// Values going over bounds are limited to the actual length.
+#[cfg(not(feature = "no_index"))]
 #[inline]
-#[allow(dead_code)]
 pub fn calc_offset_len(length: usize, start: crate::INT, len: crate::INT) -> (usize, usize) {
     let start = if start < 0 {
         let abs_start = start.unsigned_abs();

src/func/builtin.rs

@@ -4,11 +4,12 @@ use super::call::FnCallArgs;
 use super::native::FnBuiltin;
 use crate::tokenizer::{Token, Token::*};
 use crate::{
-    Dynamic, ExclusiveRange, ImmutableString, InclusiveRange, NativeCallContext, RhaiResult, INT,
+    Dynamic, ExclusiveRange, ImmutableString, InclusiveRange, NativeCallContext, RhaiResult,
+    SmartString, INT,
 };
-use std::any::TypeId;
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
+use std::{any::TypeId, fmt::Write};
 
 #[cfg(not(feature = "no_float"))]
 use crate::FLOAT;
@@ -241,7 +242,9 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
                     let x = args[0].as_char().expect(BUILTIN);
                     let y = args[1].as_char().expect(BUILTIN);
 
-                    let result = format!("{x}{y}");
+                    let mut result = SmartString::new_const();
+                    result.push(x);
+                    result.push(y);
 
                     #[cfg(not(feature = "unchecked"))]
                     _ctx.engine()
@@ -399,7 +402,10 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
             Plus => Some(|_ctx, args| {
                 let x = args[0].as_char().expect(BUILTIN);
                 let y = &*args[1].read_lock::<ImmutableString>().expect(BUILTIN);
-                let result = format!("{x}{y}");
+
+                let mut result = SmartString::new_const();
+                result.push(x);
+                result.push_str(y);
 
                 #[cfg(not(feature = "unchecked"))]
                 _ctx.engine()
@@ -679,7 +685,12 @@ pub fn get_builtin_op_assignment_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Opt
                 PlusAssign => Some(|_, args| {
                     let y = args[1].as_char().expect(BUILTIN);
                     let x = &mut *args[0].write_lock::<Dynamic>().expect(BUILTIN);
-                    Ok((*x = format!("{x}{y}").into()).into())
+
+                    let mut buf = SmartString::new_const();
+                    write!(&mut buf, "{y}").unwrap();
+                    buf.push(y);
+
+                    Ok((*x = buf.into()).into())
                 }),
                 _ => None,
             };

src/packages/array_basic.rs

@@ -247,8 +247,8 @@ pub mod array_functions {
                 let mut arr_len = array.len();
                 let mut arr = Dynamic::from_array(mem::take(array));
 
-                let (mut a1, mut m1, mut s1) = crate::Engine::calc_data_sizes(&arr, true);
+                let (mut a1, mut m1, mut s1) = arr.calc_data_sizes(true);
-                let (a2, m2, s2) = crate::Engine::calc_data_sizes(&item, true);
+                let (a2, m2, s2) = item.calc_data_sizes(true);
 
                 {
                     let mut guard = arr.write_lock::<Array>().unwrap();

src/packages/lang_core.rs

@@ -269,9 +269,13 @@ fn collect_fn_metadata(
                 .iter_script_fn()
                 .filter(|(s, a, n, p, f)| filter(*s, *a, n, *p, f))
                 .for_each(|(.., f)| list.push(make_metadata(dict, namespace.into(), f).into()));
-            for (ns, m) in module.iter_sub_modules() {
+            for (name, m) in module.iter_sub_modules() {
-                let ns = format!(
+                use std::fmt::Write;
-                    "{namespace}{}{ns}",
+
+                let mut ns = crate::SmartString::new_const();
+                write!(
+                    &mut ns,
+                    "{namespace}{}{name}",
                     crate::tokenizer::Token::DoubleColon.literal_syntax()
                 );
                 scan_module(dict, list, &ns, &**m, filter);

src/packages/string_basic.rs

@@ -1,7 +1,7 @@
 use crate::plugin::*;
-use crate::{def_package, FnPtr, INT};
+use crate::{def_package, FnPtr, SmartString, INT};
 use std::any::TypeId;
-use std::fmt::{Binary, LowerHex, Octal};
+use std::fmt::{Binary, LowerHex, Octal, Write};
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 
@@ -69,7 +69,9 @@ mod print_debug_functions {
     /// Convert the value of the `item` into a string in debug format.
     #[rhai_fn(name = "to_debug", pure)]
     pub fn to_debug_generic(ctx: NativeCallContext, item: &mut Dynamic) -> ImmutableString {
-        ctx.engine().map_type_name(&format!("{item:?}")).into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{item:?}").unwrap();
+        ctx.engine().map_type_name(&buf).into()
     }
 
     /// Return the empty string.
@@ -86,7 +88,9 @@ mod print_debug_functions {
     /// Convert the string into debug format.
     #[rhai_fn(name = "debug", name = "to_debug", pure)]
     pub fn debug_string(string: &mut ImmutableString) -> ImmutableString {
-        format!("{string:?}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{string:?}").unwrap();
+        buf.into()
     }
 
     /// Return the character into a string.
@@ -97,7 +101,9 @@ mod print_debug_functions {
     /// Convert the string into debug format.
     #[rhai_fn(name = "debug", name = "to_debug")]
     pub fn debug_char(character: char) -> ImmutableString {
-        format!("{character:?}").into()
+        let mut buf = SmartString::new_const();
+        buf.push(character);
+        buf.into()
     }
 
     /// Convert the function pointer into a string in debug format.
@@ -114,7 +120,9 @@ mod print_debug_functions {
     /// Convert the boolean value into a string in debug format.
     #[rhai_fn(name = "debug", name = "to_debug")]
     pub fn debug_bool(value: bool) -> ImmutableString {
-        format!("{value:?}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{value:?}").unwrap();
+        buf.into()
     }
 
     /// Return the empty string.
@@ -147,14 +155,18 @@ mod print_debug_functions {
     #[rhai_fn(name = "debug", name = "to_debug")]
     pub fn debug_f64(number: f64) -> ImmutableString {
         let number = crate::types::FloatWrapper::new(number);
-        format!("{number:?}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{number:?}").unwrap();
+        buf.into()
     }
     /// Convert the value of `number` into a string.
     #[cfg(not(feature = "no_float"))]
     #[rhai_fn(name = "debug", name = "to_debug")]
     pub fn debug_f32(number: f32) -> ImmutableString {
         let number = crate::types::FloatWrapper::new(number);
-        format!("{number:?}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{number:?}").unwrap();
+        buf.into()
     }
 
     /// Convert the array into a string.
@@ -217,13 +229,19 @@ mod print_debug_functions {
 #[export_module]
 mod number_formatting {
     fn to_hex<T: LowerHex>(value: T) -> ImmutableString {
-        format!("{value:x}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{value:x}").unwrap();
+        buf.into()
     }
     fn to_octal<T: Octal>(value: T) -> ImmutableString {
-        format!("{value:o}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{value:o}").unwrap();
+        buf.into()
     }
     fn to_binary<T: Binary>(value: T) -> ImmutableString {
-        format!("{value:b}").into()
+        let mut buf = SmartString::new_const();
+        write!(&mut buf, "{value:b}").unwrap();
+        buf.into()
     }
 
     /// Convert the `value` into a string in hex format.

src/packages/string_more.rs

@@ -33,7 +33,9 @@ mod string_functions {
         if s.is_empty() {
             string.clone()
         } else {
-            format!("{string}{s}").into()
+            let mut buf = SmartString::from(string.as_str());
+            buf.push_str(&s);
+            buf.into()
         }
     }
     #[rhai_fn(name = "+=", name = "append")]
@@ -41,7 +43,9 @@ mod string_functions {
         let s = print_with_func(FUNC_TO_STRING, &ctx, &mut item);
 
         if !s.is_empty() {
-            *string = format!("{string}{s}").into();
+            let mut buf = SmartString::from(string.as_str());
+            buf.push_str(&s);
+            *string = buf.into();
         }
     }
     #[rhai_fn(name = "+", pure)]
@@ -74,7 +78,10 @@ mod string_functions {
     }
     #[rhai_fn(name = "+")]
     pub fn add_prepend_char(character: char, string: &str) -> ImmutableString {
-        format!("{character}{string}").into()
+        let mut buf = SmartString::new_const();
+        buf.push(character);
+        buf.push_str(string);
+        buf.into()
     }
 
     #[rhai_fn(name = "+")]

src/parser.rs

@@ -333,8 +333,12 @@ impl ParseSettings {
 #[cfg(not(feature = "no_function"))]
 #[inline]
 #[must_use]
-pub fn make_anonymous_fn(hash: u64) -> String {
+pub fn make_anonymous_fn(hash: u64) -> Identifier {
-    format!("{}{:016x}", crate::engine::FN_ANONYMOUS, hash)
+    use std::fmt::Write;
+
+    let mut buf = Identifier::new_const();
+    write!(&mut buf, "{}{:016x}", crate::engine::FN_ANONYMOUS, hash).unwrap();
+    buf
 }
 
 /// Is this function an anonymous function?
@@ -387,7 +391,7 @@ impl Expr {
         };
 
         Err(
-            PERR::MismatchedType("a boolean expression".to_string(), type_name.to_string())
+            PERR::MismatchedType("a boolean expression".into(), type_name.into())
                 .into_err(self.start_position()),
         )
     }
@@ -405,7 +409,7 @@ impl Expr {
         };
 
         Err(
-            PERR::MismatchedType("an iterable value".to_string(), type_name.to_string())
+            PERR::MismatchedType("an iterable value".into(), type_name.into())
                 .into_err(self.start_position()),
         )
     }
@@ -426,8 +430,8 @@ fn ensure_not_statement_expr(
 fn ensure_not_assignment(input: &mut TokenStream) -> ParseResult<()> {
     match input.peek().expect(NEVER_ENDS) {
         (Token::Equals, pos) => Err(LexError::ImproperSymbol(
-            "=".to_string(),
+            "=".into(),
-            "Possibly a typo of '=='?".to_string(),
+            "Possibly a typo of '=='?".into(),
         )
         .into_err(*pos)),
         _ => Ok(()),
@@ -438,15 +442,15 @@ fn ensure_not_assignment(input: &mut TokenStream) -> ParseResult<()> {
 ///
 /// # Panics
 ///
-/// Panics if the next token is not the expected one.
+/// Panics if the next token is not the expected one, or either tokens is not a literal symbol.
 fn eat_token(input: &mut TokenStream, expected_token: Token) -> Position {
     let (t, pos) = input.next().expect(NEVER_ENDS);
 
     if t != expected_token {
         unreachable!(
             "{} expected but gets {} at {}",
-            expected_token.syntax(),
+            expected_token.literal_syntax(),
-            t.syntax(),
+            t.literal_syntax(),
             pos
         );
     }
@@ -588,7 +592,7 @@ impl Engine {
                         && !self.global_sub_modules.contains_key(root)
                     {
                         return Err(
-                            PERR::ModuleUndefined(root.to_string()).into_err(namespace.position())
+                            PERR::ModuleUndefined(root.into()).into_err(namespace.position())
                         );
                     }
 
@@ -655,8 +659,9 @@ impl Engine {
                             && !state.global_imports.iter().any(|m| m.as_str() == root)
                             && !self.global_sub_modules.contains_key(root)
                         {
-                            return Err(PERR::ModuleUndefined(root.to_string())
+                            return Err(
-                                .into_err(namespace.position()));
+                                PERR::ModuleUndefined(root.into()).into_err(namespace.position())
+                            );
                         }
 
                         namespace.set_index(index);
@@ -901,7 +906,7 @@ impl Engine {
 
             if self.max_array_size() > 0 && array.len() >= self.max_array_size() {
                 return Err(PERR::LiteralTooLarge(
-                    "Size of array literal".to_string(),
+                    "Size of array literal".into(),
                     self.max_array_size(),
                 )
                 .into_err(input.peek().expect(NEVER_ENDS).1));
@@ -1037,7 +1042,7 @@ impl Engine {
 
             if self.max_map_size() > 0 && map.len() >= self.max_map_size() {
                 return Err(PERR::LiteralTooLarge(
-                    "Number of properties in object map literal".to_string(),
+                    "Number of properties in object map literal".into(),
                     self.max_map_size(),
                 )
                 .into_err(input.peek().expect(NEVER_ENDS).1));
@@ -1185,7 +1190,7 @@ impl Engine {
                 (.., pos) => {
                     return Err(PERR::MissingToken(
                         Token::DoubleArrow.into(),
-                        "in this switch case".to_string(),
+                        "in this switch case".into(),
                     )
                     .into_err(pos))
                 }
@@ -1213,7 +1218,7 @@ impl Engine {
             } else {
                 for expr in case_expr_list {
                     let value = expr.get_literal_value().ok_or_else(|| {
-                        PERR::ExprExpected("a literal".to_string()).into_err(expr.start_position())
+                        PERR::ExprExpected("a literal".into()).into_err(expr.start_position())
                     })?;
 
                     let mut range_value: Option<RangeCase> = None;
@@ -1315,9 +1320,7 @@ impl Engine {
 
         let root_expr = match token {
             _ if !(state.expr_filter)(token) => {
-                return Err(
+                return Err(LexError::UnexpectedInput(token.to_string()).into_err(settings.pos))
-                    LexError::UnexpectedInput(token.syntax().to_string()).into_err(settings.pos)
-                )
             }
 
             Token::EOF => return Err(PERR::UnexpectedEOF.into_err(settings.pos)),
@@ -1674,11 +1677,7 @@ impl Engine {
                 token => unreachable!("Token::LexError expected but gets {:?}", token),
             },
 
-            _ => {
+            _ => return Err(LexError::UnexpectedInput(token.to_string()).into_err(settings.pos)),
-                return Err(
-                    LexError::UnexpectedInput(token.syntax().to_string()).into_err(settings.pos)
-                )
-            }
         };
 
         if !(state.expr_filter)(&input.peek().expect(NEVER_ENDS).0) {
@@ -1716,12 +1715,11 @@ impl Engine {
                 (Expr::Variable(x, ..), Token::Bang) if !x.1.is_empty() => {
                     return match input.peek().expect(NEVER_ENDS) {
                         (Token::LeftParen | Token::Unit, ..) => {
-                            Err(LexError::UnexpectedInput(Token::Bang.syntax().to_string())
+                            Err(LexError::UnexpectedInput(Token::Bang.into()).into_err(tail_pos))
-                                .into_err(tail_pos))
                         }
                         _ => Err(LexError::ImproperSymbol(
-                            "!".to_string(),
+                            "!".into(),
-                            "'!' cannot be used to call module functions".to_string(),
+                            "'!' cannot be used to call module functions".into(),
                         )
                         .into_err(tail_pos)),
                     };
@@ -1732,7 +1730,7 @@ impl Engine {
                         (Token::LeftParen | Token::Unit, ..) => (),
                         (_, pos) => {
                             return Err(PERR::MissingToken(
-                                Token::LeftParen.syntax().into(),
+                                Token::LeftParen.into(),
                                 "to start arguments list of function call".into(),
                             )
                             .into_err(*pos))
@@ -1821,11 +1819,9 @@ impl Engine {
                     Self::make_dot_expr(state, expr, rhs, ASTFlags::NONE, op_flags, tail_pos)?
                 }
                 // Unknown postfix operator
-                (expr, token) => unreachable!(
+                (expr, token) => {
-                    "unknown postfix operator '{}' for {:?}",
+                    unreachable!("unknown postfix operator '{}' for {:?}", token, expr)
-                    token.syntax(),
+                }
-                    expr
-                ),
             }
         }
 
@@ -1863,7 +1859,7 @@ impl Engine {
                         && !self.global_sub_modules.contains_key(root)
                     {
                         return Err(
-                            PERR::ModuleUndefined(root.to_string()).into_err(namespace.position())
+                            PERR::ModuleUndefined(root.into()).into_err(namespace.position())
                         );
                     }
 
@@ -1890,7 +1886,7 @@ impl Engine {
         let (token, token_pos) = input.peek().expect(NEVER_ENDS);
 
         if !(state.expr_filter)(token) {
-            return Err(LexError::UnexpectedInput(token.syntax().to_string()).into_err(*token_pos));
+            return Err(LexError::UnexpectedInput(token.to_string()).into_err(*token_pos));
         }
 
         let mut settings = settings;
@@ -2092,8 +2088,8 @@ impl Engine {
             }
             // ??? && ??? = rhs, ??? || ??? = rhs, xxx ?? xxx = rhs
             Expr::And(..) | Expr::Or(..) | Expr::Coalesce(..) => Err(LexError::ImproperSymbol(
-                "=".to_string(),
+                "=".into(),
-                "Possibly a typo of '=='?".to_string(),
+                "Possibly a typo of '=='?".into(),
             )
             .into_err(op_pos)),
             // expr = rhs
@@ -2362,7 +2358,7 @@ impl Engine {
             #[cfg(not(feature = "unchecked"))]
             settings.ensure_level_within_max_limit(state.max_expr_depth)?;
 
-            let op = op_token.syntax();
+            let op = op_token.to_string();
             let hash = calc_fn_hash(None, &op, 2);
             let is_valid_script_function = is_valid_function_name(&op);
             let operator_token = if is_valid_script_function {
@@ -2379,7 +2375,7 @@ impl Engine {
             let mut op_base = FnCallExpr {
                 #[cfg(not(feature = "no_module"))]
                 namespace: Default::default(),
-                name: state.get_interned_string(op.as_ref()),
+                name: state.get_interned_string(&op),
                 hashes: FnCallHashes::from_native(hash),
                 args,
                 op_token: operator_token,
@@ -2492,7 +2488,7 @@ impl Engine {
             settings.pos = *fwd_pos;
             let settings = settings.level_up();
 
-            required_token = match parse_func(&segments, &*fwd_token.syntax(), &mut user_state) {
+            required_token = match parse_func(&segments, &fwd_token.to_string(), &mut user_state) {
                 Ok(Some(seg))
                     if seg.starts_with(CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT)
                         && seg.len() > CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT.len() =>
@@ -2551,8 +2547,7 @@ impl Engine {
                     }
                     (.., pos) => {
                         return Err(
-                            PERR::MissingSymbol("Expecting 'true' or 'false'".to_string())
+                            PERR::MissingSymbol("Expecting 'true' or 'false'".into()).into_err(pos)
-                                .into_err(pos),
                         )
                     }
                 },
@@ -2564,8 +2559,7 @@ impl Engine {
                     }
                     (.., pos) => {
                         return Err(
-                            PERR::MissingSymbol("Expecting an integer number".to_string())
+                            PERR::MissingSymbol("Expecting an integer number".into()).into_err(pos)
-                                .into_err(pos),
                         )
                     }
                 },
@@ -2577,10 +2571,10 @@ impl Engine {
                         tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_FLOAT));
                     }
                     (.., pos) => {
-                        return Err(PERR::MissingSymbol(
+                        return Err(
-                            "Expecting a floating-point number".to_string(),
+                            PERR::MissingSymbol("Expecting a floating-point number".into())
+                                .into_err(pos),
                         )
-                        .into_err(pos))
                     }
                 },
                 CUSTOM_SYNTAX_MARKER_STRING => match input.next().expect(NEVER_ENDS) {
@@ -2591,20 +2585,26 @@ impl Engine {
                         tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_STRING));
                     }
                     (.., pos) => {
-                        return Err(
+                        return Err(PERR::MissingSymbol("Expecting a string".into()).into_err(pos))
-                            PERR::MissingSymbol("Expecting a string".to_string()).into_err(pos)
-                        )
                     }
                 },
                 s => match input.next().expect(NEVER_ENDS) {
                     (Token::LexError(err), pos) => return Err(err.into_err(pos)),
-                    (t, ..) if &*t.syntax() == s => {
+                    (Token::Identifier(t), ..)
+                    | (Token::Reserved(t), ..)
+                    | (Token::Custom(t), ..)
+                        if *t == s =>
+                    {
+                        segments.push(required_token.clone());
+                        tokens.push(required_token.clone().into());
+                    }
+                    (t, ..) if t.is_literal() && t.literal_syntax() == s => {
                         segments.push(required_token.clone());
                         tokens.push(required_token.clone().into());
                     }
                     (.., pos) => {
                         return Err(PERR::MissingToken(
-                            s.to_string(),
+                            s.into(),
                             format!("for '{}' expression", segments[0]),
                         )
                         .into_err(pos))
@@ -2805,9 +2805,7 @@ impl Engine {
             let (counter_name, counter_pos) = parse_var_name(input)?;
 
             if counter_name == name {
-                return Err(
+                return Err(PERR::DuplicatedVariable(counter_name.into()).into_err(counter_pos));
-                    PERR::DuplicatedVariable(counter_name.to_string()).into_err(counter_pos)
-                );
             }
 
             let (has_close_paren, pos) = match_token(input, Token::RightParen);
@@ -2893,7 +2891,7 @@ impl Engine {
         let (name, pos) = parse_var_name(input)?;
 
         if !self.allow_shadowing() && state.stack.iter().any(|(v, ..)| v == name) {
-            return Err(PERR::VariableExists(name.to_string()).into_err(pos));
+            return Err(PERR::VariableExists(name.into()).into_err(pos));
         }
 
         if let Some(ref filter) = self.def_var_filter {
@@ -2914,10 +2912,10 @@ impl Engine {
 
             match filter(false, info, context) {
                 Ok(true) => (),
-                Ok(false) => return Err(PERR::ForbiddenVariable(name.to_string()).into_err(pos)),
+                Ok(false) => return Err(PERR::ForbiddenVariable(name.into()).into_err(pos)),
                 Err(err) => match *err {
                     EvalAltResult::ErrorParsing(e, pos) => return Err(e.into_err(pos)),
-                    _ => return Err(PERR::ForbiddenVariable(name.to_string()).into_err(pos)),
+                    _ => return Err(PERR::ForbiddenVariable(name.into()).into_err(pos)),
                 },
             }
         }
@@ -3361,7 +3359,7 @@ impl Engine {
 
                     (.., pos) => Err(PERR::MissingToken(
                         Token::Fn.into(),
-                        format!("following '{}'", Token::Private.syntax()),
+                        format!("following '{}'", Token::Private),
                     )
                     .into_err(pos)),
                 }
@@ -3564,7 +3562,7 @@ impl Engine {
                 eat_token(input, Token::Unit);
                 true
             }
-            (.., pos) => return Err(PERR::FnMissingParams(name.to_string()).into_err(*pos)),
+            (.., pos) => return Err(PERR::FnMissingParams(name.into()).into_err(*pos)),
         };
 
         let mut params = StaticVec::<(ImmutableString, _)>::new_const();
@@ -3577,8 +3575,9 @@ impl Engine {
                     (Token::RightParen, ..) => break,
                     (Token::Identifier(s), pos) => {
                         if params.iter().any(|(p, _)| p.as_str() == &*s) {
-                            return Err(PERR::FnDuplicatedParam(name.to_string(), s.to_string())
+                            return Err(
-                                .into_err(pos));
+                                PERR::FnDuplicatedParam(name.into(), s.to_string()).into_err(pos)
+                            );
                         }
                         let s = state.get_interned_string(*s);
                         state.stack.push(s.clone(), ());
@@ -3611,7 +3610,7 @@ impl Engine {
                 settings.is_breakable = false;
                 self.parse_block(input, state, lib, settings.level_up())?
             }
-            (.., pos) => return Err(PERR::FnMissingBody(name.to_string()).into_err(*pos)),
+            (.., pos) => return Err(PERR::FnMissingBody(name.into()).into_err(*pos)),
         }
         .into();
 
@@ -3851,9 +3850,7 @@ impl Engine {
         match input.peek().expect(NEVER_ENDS) {
             (Token::EOF, ..) => (),
             // Return error if the expression doesn't end
-            (token, pos) => {
+            (token, pos) => return Err(LexError::UnexpectedInput(token.to_string()).into_err(*pos)),
-                return Err(LexError::UnexpectedInput(token.syntax().to_string()).into_err(*pos))
-            }
         }
 
         let mut statements = StmtBlockContainer::new_const();

src/tokenizer.rs

@@ -9,7 +9,6 @@ use crate::{Engine, Identifier, LexError, SmartString, StaticVec, INT, UNSIGNED_
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 use std::{
-    borrow::Cow,
     cell::RefCell,
     char, fmt,
     iter::{FusedIterator, Peekable},
@@ -591,12 +590,62 @@ pub enum Token {
 impl fmt::Display for Token {
     #[inline(always)]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&self.syntax())
+        use Token::*;
+
+        match self {
+            IntegerConstant(i) => write!(f, "{i}"),
+            #[cfg(not(feature = "no_float"))]
+            FloatConstant(v) => write!(f, "{v}"),
+            #[cfg(feature = "decimal")]
+            DecimalConstant(d) => write!(f, "{d}"),
+            StringConstant(s) => write!(f, r#""{s}""#),
+            InterpolatedString(..) => f.write_str("string"),
+            CharConstant(c) => write!(f, "{c}"),
+            Identifier(s) => f.write_str(s),
+            Reserved(s) => f.write_str(s),
+            #[cfg(not(feature = "no_custom_syntax"))]
+            Custom(s) => f.write_str(s),
+            LexError(err) => write!(f, "{err}"),
+            Comment(s) => f.write_str(s),
+
+            EOF => f.write_str("{EOF}"),
+
+            token => f.write_str(token.literal_syntax()),
+        }
     }
 }
 
 impl Token {
+    /// Is the token a literal symbol?
+    #[must_use]
+    pub const fn is_literal(&self) -> bool {
+        use Token::*;
+
+        match self {
+            IntegerConstant(..) => false,
+            #[cfg(not(feature = "no_float"))]
+            FloatConstant(..) => false,
+            #[cfg(feature = "decimal")]
+            DecimalConstant(..) => false,
+            StringConstant(..)
+            | InterpolatedString(..)
+            | CharConstant(..)
+            | Identifier(..)
+            | Reserved(..) => false,
+            #[cfg(not(feature = "no_custom_syntax"))]
+            Custom(..) => false,
+            LexError(..) | Comment(..) => false,
+
+            EOF => false,
+
+            _ => true,
+        }
+    }
     /// Get the literal syntax of the token.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the token is not a literal symbol.
     #[must_use]
     pub const fn literal_syntax(&self) -> &'static str {
         use Token::*;
@@ -690,34 +739,7 @@ impl Token {
             #[cfg(not(feature = "no_module"))]
             As => "as",
 
-            _ => "ERROR: NOT A KEYWORD",
+            _ => panic!("token is not a literal symbol"),
-        }
-    }
-
-    /// Get the syntax of the token.
-    #[must_use]
-    pub fn syntax(&self) -> Cow<'static, str> {
-        use Token::*;
-
-        match self {
-            IntegerConstant(i) => i.to_string().into(),
-            #[cfg(not(feature = "no_float"))]
-            FloatConstant(f) => f.to_string().into(),
-            #[cfg(feature = "decimal")]
-            DecimalConstant(d) => d.to_string().into(),
-            StringConstant(s) => format!("\"{s}\"").into(),
-            InterpolatedString(..) => "string".into(),
-            CharConstant(c) => c.to_string().into(),
-            Identifier(s) => s.to_string().into(),
-            Reserved(s) => s.to_string().into(),
-            #[cfg(not(feature = "no_custom_syntax"))]
-            Custom(s) => s.to_string().into(),
-            LexError(err) => err.to_string().into(),
-            Comment(s) => s.to_string().into(),
-
-            EOF => "{EOF}".into(),
-
-            token => token.literal_syntax().into(),
         }
     }
 
@@ -1127,7 +1149,7 @@ impl Token {
 impl From<Token> for String {
     #[inline(always)]
     fn from(token: Token) -> Self {
-        token.syntax().into()
+        token.to_string()
     }
 }
 
@@ -2399,7 +2421,7 @@ impl<'a> Iterator for TokenIterator<'a> {
             Some((Token::Reserved(s), pos)) => (match
                 (s.as_str(),
                     #[cfg(not(feature = "no_custom_syntax"))]
-                    (!self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(&*s)),
+                    self.engine.custom_keywords.contains_key(&*s),
                     #[cfg(feature = "no_custom_syntax")]
                     false
                 )
@@ -2436,7 +2458,7 @@ impl<'a> Iterator for TokenIterator<'a> {
                 #[cfg(feature = "no_custom_syntax")]
                 (.., true) => unreachable!("no custom operators"),
                 // Reserved keyword that is not custom and disabled.
-                (token, false) if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token) => {
+                (token, false) if self.engine.disabled_symbols.contains(token) => {
                     let msg = format!("reserved {} '{token}' is disabled", if is_valid_identifier(token) { "keyword"} else {"symbol"});
                     Token::LexError(LERR::ImproperSymbol(s.to_string(), msg).into())
                 },
@@ -2445,13 +2467,13 @@ impl<'a> Iterator for TokenIterator<'a> {
             }, pos),
             // Custom keyword
             #[cfg(not(feature = "no_custom_syntax"))]
-            Some((Token::Identifier(s), pos)) if !self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(&*s) => {
+            Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&*s) => {
                 (Token::Custom(s), pos)
             }
             // Custom keyword/symbol - must be disabled
             #[cfg(not(feature = "no_custom_syntax"))]
-            Some((token, pos)) if !self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
+            Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
-                if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token.literal_syntax()) {
+                if self.engine.disabled_symbols.contains(token.literal_syntax()) {
                     // Disabled standard keyword/symbol
                     (Token::Custom(Box::new(token.literal_syntax().into())), pos)
                 } else {
@@ -2460,7 +2482,7 @@ impl<'a> Iterator for TokenIterator<'a> {
                 }
             }
             // Disabled symbol
-            Some((token, pos)) if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token.literal_syntax()) => {
+            Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.contains(token.literal_syntax()) => {
                 (Token::Reserved(Box::new(token.literal_syntax().into())), pos)
             }
             // Normal symbol