Use SmartString in parsing.

src/ast/flags.rs

@@ -4,15 +4,6 @@ use bitflags::bitflags;
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 
-/// A type representing the access mode of a function.
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
-pub enum FnAccess {
-    /// Private function.
-    Private,
-    /// Public function.
-    Public,
-}
-
 bitflags! {
     /// _(internals)_ A type that holds a configuration option with bit-flags.
     /// Exported under the `internals` feature only.

src/ast/mod.rs

@@ -9,13 +9,13 @@ pub mod stmt;
 
 pub use ast::{ASTNode, AST};
 pub use expr::{BinaryExpr, CustomExpr, Expr, FnCallExpr, FnCallHashes};
-pub use flags::{ASTFlags, FnAccess};
+pub use flags::ASTFlags;
 pub use ident::Ident;
 #[cfg(not(feature = "no_module"))]
 #[cfg(not(feature = "no_function"))]
 pub use script_fn::EncapsulatedEnviron;
 #[cfg(not(feature = "no_function"))]
-pub use script_fn::{ScriptFnDef, ScriptFnMetadata};
+pub use script_fn::{FnAccess, ScriptFnDef, ScriptFnMetadata};
 pub use stmt::{
     ConditionalStmtBlock, OpAssignment, Stmt, StmtBlock, StmtBlockContainer, SwitchCases,
     TryCatchBlock,

src/ast/script_fn.rs

@@ -1,12 +1,21 @@
 //! Module defining script-defined functions.
 #![cfg(not(feature = "no_function"))]
 
-use super::{FnAccess, StmtBlock};
+use super::StmtBlock;
 use crate::{Identifier, StaticVec};
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 use std::{fmt, hash::Hash};
 
+/// A type representing the access mode of a function.
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum FnAccess {
+    /// Private function.
+    Private,
+    /// Public function.
+    Public,
+}
+
 /// _(internals)_ Encapsulated AST environment.
 /// Exported under the `internals` feature only.
 ///

src/lib.rs

@@ -365,7 +365,7 @@ pub type StaticVec<T> = smallvec::SmallVec<[T; 3]>;
 ///
 /// Under `no_closure`, this type aliases to [`StaticVec`][crate::StaticVec] instead.
 #[cfg(not(feature = "no_closure"))]
-type FnArgsVec<T> = smallvec::SmallVec<[T; 8]>;
+type FnArgsVec<T> = smallvec::SmallVec<[T; 5]>;
 
 /// Inline arguments storage for function calls.
 /// This type aliases to [`StaticVec`][crate::StaticVec].

src/parser.rs

@@ -19,7 +19,7 @@ use crate::types::StringsInterner;
 use crate::{
     calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, ExclusiveRange, Identifier,
     ImmutableString, InclusiveRange, LexError, OptimizationLevel, ParseError, Position, Scope,
-    Shared, StaticVec, AST, INT, PERR,
+    Shared, SmartString, StaticVec, AST, INT, PERR,
 };
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
@@ -387,7 +387,7 @@ fn match_token(input: &mut TokenStream, token: Token) -> (bool, Position) {
 }
 
 /// Parse a variable name.
-fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)> {
+fn parse_var_name(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> {
     match input.next().expect(NEVER_ENDS) {
         // Variable name
         (Token::Identifier(s), pos) => Ok((s, pos)),
@@ -403,7 +403,7 @@ fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)>
 }
 
 /// Parse a symbol.
-fn parse_symbol(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)> {
+fn parse_symbol(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> {
     match input.next().expect(NEVER_ENDS) {
         // Symbol
         (token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)),
@@ -2059,7 +2059,7 @@ fn parse_binary_op(
             Token::Custom(c) => state
                 .engine
                 .custom_keywords
-                .get(c.as_ref())
+                .get(c)
                 .cloned()
                 .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*current_pos))?,
             Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
@@ -2084,7 +2084,7 @@ fn parse_binary_op(
             Token::Custom(c) => state
                 .engine
                 .custom_keywords
-                .get(c.as_ref())
+                .get(c)
                 .cloned()
                 .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*next_pos))?,
             Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
@@ -2182,7 +2182,7 @@ fn parse_binary_op(
                 if state
                     .engine
                     .custom_keywords
-                    .get(s.as_ref())
+                    .get(s.as_str())
                     .map_or(false, Option::is_some) =>
             {
                 let hash = calc_fn_hash(&s, 2);
@@ -2630,14 +2630,12 @@ fn parse_let(
     // let name ...
     let (name, pos) = parse_var_name(input)?;
 
-    if !settings.default_options.allow_shadowing
+    if !settings.default_options.allow_shadowing && state.stack.iter().any(|(v, ..)| v == &name) {
-        && state.stack.iter().any(|(v, ..)| v == name.as_ref())
-    {
         return Err(PERR::VariableExists(name.to_string()).into_err(pos));
     }
 
     if let Some(ref filter) = state.engine.def_var_filter {
-        let will_shadow = state.stack.iter().any(|(v, ..)| v == name.as_ref());
+        let will_shadow = state.stack.iter().any(|(v, ..)| v == &name);
         let level = settings.level;
         let is_const = access == AccessMode::ReadOnly;
         let info = VarDefInfo {
@@ -2820,7 +2818,7 @@ fn parse_block(
         }
     };
 
-    let mut statements = Vec::with_capacity(8);
+    let mut statements = StaticVec::new();
 
     let prev_entry_stack_len = state.block_stack_len;
     state.block_stack_len = state.stack.len();
@@ -2924,7 +2922,7 @@ fn parse_stmt(
     #[cfg(not(feature = "no_function"))]
     #[cfg(feature = "metadata")]
     let comments = {
-        let mut comments = Vec::<Box<str>>::new();
+        let mut comments = StaticVec::<SmartString>::new();
         let mut comments_pos = Position::NONE;
 
         // Handle doc-comments.
@@ -3201,7 +3199,7 @@ fn parse_fn(
     settings: ParseSettings,
     #[cfg(not(feature = "no_function"))]
     #[cfg(feature = "metadata")]
-    comments: Vec<Box<str>>,
+    comments: StaticVec<SmartString>,
 ) -> ParseResult<ScriptFnDef> {
     #[cfg(not(feature = "unchecked"))]
     settings.ensure_level_within_max_limit(state.max_expr_depth)?;
@@ -3285,7 +3283,13 @@ fn parse_fn(
         comments: if comments.is_empty() {
             None
         } else {
-            Some(comments.into())
+            Some(
+                comments
+                    .into_iter()
+                    .map(|s| s.to_string().into_boxed_str())
+                    .collect::<Vec<_>>()
+                    .into_boxed_slice(),
+            )
         },
     })
 }

src/tokenizer.rs

@@ -5,7 +5,7 @@ use crate::engine::{
     KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
 };
 use crate::func::native::OnParseTokenCallback;
-use crate::{Engine, LexError, StaticVec, INT, UNSIGNED_INT};
+use crate::{Engine, LexError, SmartString, StaticVec, INT, UNSIGNED_INT};
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 use std::{
@@ -363,13 +363,13 @@ pub enum Token {
     #[cfg(feature = "decimal")]
     DecimalConstant(rust_decimal::Decimal),
     /// An identifier.
-    Identifier(Box<str>),
+    Identifier(SmartString),
     /// A character constant.
     CharConstant(char),
     /// A string constant.
-    StringConstant(Box<str>),
+    StringConstant(SmartString),
     /// An interpolated string.
-    InterpolatedString(Box<str>),
+    InterpolatedString(SmartString),
     /// `{`
     LeftBrace,
     /// `}`
@@ -536,11 +536,11 @@ pub enum Token {
     /// A lexer error.
     LexError(LexError),
     /// A comment block.
-    Comment(Box<str>),
+    Comment(SmartString),
     /// A reserved symbol.
-    Reserved(Box<str>),
+    Reserved(SmartString),
     /// A custom keyword.
-    Custom(Box<str>),
+    Custom(SmartString),
     /// End of the input stream.
     EOF,
 }
@@ -1022,9 +1022,9 @@ impl Token {
     /// Convert a token into a function name, if possible.
     #[cfg(not(feature = "no_function"))]
     #[inline]
-    pub(crate) fn into_function_name_for_override(self) -> Result<Box<str>, Self> {
+    pub(crate) fn into_function_name_for_override(self) -> Result<SmartString, Self> {
         match self {
-            Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&*s) => Ok(s),
+            Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&s) => Ok(s),
             _ => Err(self),
         }
     }
@@ -1112,9 +1112,9 @@ pub fn parse_string_literal(
     verbatim: bool,
     allow_line_continuation: bool,
     allow_interpolation: bool,
-) -> Result<(Box<str>, bool, Position), (LexError, Position)> {
+) -> Result<(SmartString, bool, Position), (LexError, Position)> {
-    let mut result = String::with_capacity(12);
+    let mut result = SmartString::new();
-    let mut escape = String::with_capacity(12);
+    let mut escape = SmartString::new();
 
     let start = *pos;
     let mut first_char = Position::NONE;
@@ -1146,7 +1146,6 @@ pub fn parse_string_literal(
                 break;
             }
             None => {
-                result += &escape;
                 pos.advance();
                 state.is_within_text_terminated_by = None;
                 return Err((LERR::UnterminatedString, start));
@@ -1242,7 +1241,7 @@ pub fn parse_string_literal(
 
                 result.push(
                     char::from_u32(out_val)
-                        .ok_or_else(|| (LERR::MalformedEscapeSequence(seq), *pos))?,
+                        .ok_or_else(|| (LERR::MalformedEscapeSequence(seq.to_string()), *pos))?,
                 );
             }
 
@@ -1283,7 +1282,7 @@ pub fn parse_string_literal(
             _ if !escape.is_empty() => {
                 escape.push(next_char);
 
-                return Err((LERR::MalformedEscapeSequence(escape), *pos));
+                return Err((LERR::MalformedEscapeSequence(escape.to_string()), *pos));
             }
 
             // Whitespace to skip
@@ -1309,7 +1308,7 @@ pub fn parse_string_literal(
         }
     }
 
-    Ok((result.into(), interpolated, first_char))
+    Ok((result, interpolated, first_char))
 }
 
 /// Consume the next character.
@@ -2291,18 +2290,18 @@ impl<'a> Iterator for TokenIterator<'a> {
                 (Token::Custom(s), pos)
             }
             // Custom keyword/symbol - must be disabled
-            Some((token, pos)) if self.engine.custom_keywords.contains_key(&*token.syntax()) => {
+            Some((token, pos)) if self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
-                if self.engine.disabled_symbols.contains(&*token.syntax()) {
+                if self.engine.disabled_symbols.contains(token.literal_syntax()) {
                     // Disabled standard keyword/symbol
-                    (Token::Custom(token.syntax().into()), pos)
+                    (Token::Custom(token.literal_syntax().into()), pos)
                 } else {
                     // Active standard keyword - should never be a custom keyword!
                     unreachable!("{:?} is an active keyword", token)
                 }
             }
             // Disabled symbol
-            Some((token, pos)) if self.engine.disabled_symbols.contains(&*token.syntax()) => {
+            Some((token, pos)) if self.engine.disabled_symbols.contains(token.literal_syntax()) => {
-                (Token::Reserved(token.syntax().into()), pos)
+                (Token::Reserved(token.literal_syntax().into()), pos)
             }
             // Normal symbol
             Some(r) => r,